mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-16 08:18:29 -04:00
feat(nodes): add fleet operations dashboard (#12046)
* feat(nodes): report CPU telemetry Assisted-by: Codex:gpt-6 * feat(nodes): add fleet view utilities Assisted-by: Codex:gpt-6 * feat(nodes): add fleet operations dashboard Replace the panel roster with aggregate capacity gauges, fleet filtering and selection, bounded bulk actions, and an on-demand node inspector. Extend node details and distributed-mode documentation with CPU and models-disk telemetry. Assisted-by: Codex:gpt-6 * fix(nodes): harden fleet lifecycle actions Assisted-by: Codex:gpt-6 * fix(nodes): restore compact fleet composition Keep fleet health, capacity, and attention in one compact overview at ordinary desktop widths. The inspector now overlays the roster until the workbench can preserve a useful table beside it. Assisted-by: Codex:gpt-6 * feat(nodes): add accessible running models workbench Assisted-by: Codex:gpt-6 * fix(nodes): correct model view ARIA links Keep each tab panel available for its controlling tab while native hidden state removes inactive content from accessibility navigation. Model controls now expose only supported state and valid inspector relationships. Assisted-by: Codex:gpt-6 * fix(nodes): align lifecycle and capacity states Pending nodes now expose approval wherever node actions appear, while other lifecycle controls follow the server transition rules. Capacity totals exclude incomplete readings so missing availability remains unknown. Assisted-by: Codex:gpt-6 * fix(nodes): restore approved dashboard composition Assisted-by: Codex:gpt-6 * fix(nodes): integrate operate navigation Assisted-by: Codex:gpt-6 * fix(nodes): restore low density fleet view Assisted-by: Codex:gpt-6 * fix(nodes): preserve complete operate menu Assisted-by: Codex:gpt-6 * fix(nodes): preserve inspector workspace height Assisted-by: Codex:gpt-6 * fix(nodes): restore standard operate navigation Assisted-by: Codex:gpt-6 * feat(ui): add collapsible console rail Assisted-by: Codex:gpt-6 * feat(nodes): stop models from fleet view Assisted-by: Codex:gpt-6 * fix(nodes): make inspector a full height drawer Assisted-by: Codex:gpt-6 * fix(model): stop mixed local and remote placements Assisted-by: Codex:gpt-6 * fix(ui): announce action menu navigation Assisted-by: Codex:gpt-6 * fix(nodes): keep inspector within viewport Assisted-by: Codex:gpt-6 * fix(ui): preserve focus across model actions Assisted-by: Codex:gpt-6 --------- Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
localai-org-maint-botandEttore Di Giacinto authored and GitHub committed
1 parent
f9dab888fe
commit
997d403de4
39 files changed
+3384
-439
No files matched your search
@@ -75,15 +75,18 @@ func GetNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc {
|
||||
|
||||
// RegisterNodeRequest is the request body for registering a new worker node.
|
||||
type RegisterNodeRequest struct {
|
||||
Name string `json:"name"`
|
||||
NodeType string `json:"node_type,omitempty"` // "backend" (default) or "agent"
|
||||
Address string `json:"address"`
|
||||
HTTPAddress string `json:"http_address,omitempty"`
|
||||
Token string `json:"token,omitempty"`
|
||||
TotalVRAM uint64 `json:"total_vram,omitempty"`
|
||||
AvailableVRAM uint64 `json:"available_vram,omitempty"`
|
||||
TotalRAM uint64 `json:"total_ram,omitempty"`
|
||||
AvailableRAM uint64 `json:"available_ram,omitempty"`
|
||||
Name string `json:"name"`
|
||||
NodeType string `json:"node_type,omitempty"` // "backend" (default) or "agent"
|
||||
Address string `json:"address"`
|
||||
HTTPAddress string `json:"http_address,omitempty"`
|
||||
Token string `json:"token,omitempty"`
|
||||
TotalVRAM uint64 `json:"total_vram,omitempty"`
|
||||
AvailableVRAM uint64 `json:"available_vram,omitempty"`
|
||||
TotalRAM uint64 `json:"total_ram,omitempty"`
|
||||
AvailableRAM uint64 `json:"available_ram,omitempty"`
|
||||
CPULogicalCores uint64 `json:"cpu_logical_cores,omitempty"`
|
||||
CPUUsagePercent float64 `json:"cpu_usage_percent,omitempty"`
|
||||
CPULoad1 float64 `json:"cpu_load_1,omitempty"`
|
||||
// TotalDisk / AvailableDisk describe the filesystem backing the worker's
|
||||
// MODELS directory (where staged weights land), not the root filesystem.
|
||||
// Omitted by workers that predate the fields; the scheduler treats
|
||||
@@ -182,6 +185,9 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au
|
||||
AvailableVRAM: req.AvailableVRAM,
|
||||
TotalRAM: req.TotalRAM,
|
||||
AvailableRAM: req.AvailableRAM,
|
||||
CPULogicalCores: req.CPULogicalCores,
|
||||
CPUUsagePercent: req.CPUUsagePercent,
|
||||
CPULoad1: req.CPULoad1,
|
||||
TotalDisk: req.TotalDisk,
|
||||
AvailableDisk: req.AvailableDisk,
|
||||
GPUVendor: req.GPUVendor,
|
||||
@@ -381,7 +387,8 @@ func HeartbeatEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc {
|
||||
|
||||
var updatePtr *nodes.HeartbeatUpdate
|
||||
if update.AvailableVRAM != nil || update.TotalVRAM != nil || update.AvailableRAM != nil ||
|
||||
update.AvailableDisk != nil || update.TotalDisk != nil || update.GPUVendor != "" {
|
||||
update.AvailableDisk != nil || update.TotalDisk != nil || update.GPUVendor != "" ||
|
||||
update.CPUUsagePercent != nil || update.CPULoad1 != nil {
|
||||
updatePtr = &update
|
||||
}
|
||||
|
||||
@@ -431,6 +438,12 @@ func DrainNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc {
|
||||
ctx := c.Request().Context()
|
||||
id := c.Param("id")
|
||||
if err := registry.MarkDraining(ctx, id); err != nil {
|
||||
if errors.Is(err, nodes.ErrNodeNotFound) {
|
||||
return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found"))
|
||||
}
|
||||
if errors.Is(err, nodes.ErrNodeStatusConflict) {
|
||||
return c.JSON(http.StatusConflict, nodeError(http.StatusConflict, "node must be healthy to drain"))
|
||||
}
|
||||
xlog.Error("Failed to drain node", "id", id, "error", err)
|
||||
return c.JSON(http.StatusInternalServerError, nodeError(http.StatusInternalServerError, "failed to drain node"))
|
||||
}
|
||||
@@ -443,7 +456,13 @@ func ResumeNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
ctx := c.Request().Context()
|
||||
id := c.Param("id")
|
||||
if err := registry.MarkHealthy(ctx, id); err != nil {
|
||||
if err := registry.ResumeNode(ctx, id); err != nil {
|
||||
if errors.Is(err, nodes.ErrNodeNotFound) {
|
||||
return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found"))
|
||||
}
|
||||
if errors.Is(err, nodes.ErrNodeStatusConflict) {
|
||||
return c.JSON(http.StatusConflict, nodeError(http.StatusConflict, "node must be draining to resume"))
|
||||
}
|
||||
xlog.Error("Failed to resume node", "id", id, "error", err)
|
||||
return c.JSON(http.StatusInternalServerError, nodeError(http.StatusInternalServerError, "failed to resume node"))
|
||||
}
|
||||
|
||||
@@ -58,6 +58,35 @@ var _ = Describe("Node HTTP handlers", func() {
|
||||
})
|
||||
|
||||
Describe("RegisterNodeEndpoint", func() {
|
||||
It("binds, persists, and lists CPU telemetry", func() {
|
||||
e := echo.New()
|
||||
body := `{"name":"cpu-worker","address":"10.0.0.9:50051","cpu_logical_cores":12,"cpu_usage_percent":145,"cpu_load_1":2.5}`
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
Expect(RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{})(e.NewContext(req, rec))).To(Succeed())
|
||||
Expect(rec.Code).To(Equal(http.StatusCreated))
|
||||
|
||||
node, err := registry.GetByName(context.Background(), "cpu-worker")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(node.CPULogicalCores).To(Equal(uint64(12)))
|
||||
Expect(node.CPUUsagePercent).To(Equal(float64(100)))
|
||||
Expect(node.CPULoad1).To(Equal(2.5))
|
||||
|
||||
listRecorder := httptest.NewRecorder()
|
||||
listContext := e.NewContext(httptest.NewRequest(http.MethodGet, "/api/nodes", nil), listRecorder)
|
||||
Expect(ListNodesEndpoint(registry)(listContext)).To(Succeed())
|
||||
var listed []map[string]any
|
||||
Expect(json.Unmarshal(listRecorder.Body.Bytes(), &listed)).To(Succeed())
|
||||
Expect(listed).To(HaveLen(1))
|
||||
Expect(listed[0]).To(SatisfyAll(
|
||||
HaveKeyWithValue("cpu_logical_cores", float64(12)),
|
||||
HaveKeyWithValue("cpu_usage_percent", float64(100)),
|
||||
HaveKeyWithValue("cpu_load_1", 2.5),
|
||||
))
|
||||
})
|
||||
|
||||
It("registers a backend node and returns 201", func() {
|
||||
e := echo.New()
|
||||
body := `{"name":"worker-1","address":"10.0.0.1:50051"}`
|
||||
@@ -449,6 +478,42 @@ var _ = Describe("Node HTTP handlers", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Node lifecycle endpoints", func() {
|
||||
request := func(handler echo.HandlerFunc, id string) *httptest.ResponseRecorder {
|
||||
e := echo.New()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/nodes/"+id, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
c := e.NewContext(req, rec)
|
||||
c.SetParamNames("id")
|
||||
c.SetParamValues(id)
|
||||
Expect(handler(c)).To(Succeed())
|
||||
return rec
|
||||
}
|
||||
|
||||
It("accepts healthy drain followed by draining resume", func() {
|
||||
Expect(registry.Register(context.Background(), &nodes.BackendNode{
|
||||
ID: "lifecycle", Name: "lifecycle", Address: "10.0.0.10:50051",
|
||||
}, true)).To(Succeed())
|
||||
|
||||
Expect(request(DrainNodeEndpoint(registry), "lifecycle").Code).To(Equal(http.StatusOK))
|
||||
Expect(request(ResumeNodeEndpoint(registry), "lifecycle").Code).To(Equal(http.StatusOK))
|
||||
})
|
||||
|
||||
It("returns conflict when a pending node is drained or resumed", func() {
|
||||
Expect(registry.Register(context.Background(), &nodes.BackendNode{
|
||||
ID: "pending-lifecycle", Name: "pending-lifecycle", Address: "10.0.0.11:50051",
|
||||
}, false)).To(Succeed())
|
||||
|
||||
Expect(request(DrainNodeEndpoint(registry), "pending-lifecycle").Code).To(Equal(http.StatusConflict))
|
||||
Expect(request(ResumeNodeEndpoint(registry), "pending-lifecycle").Code).To(Equal(http.StatusConflict))
|
||||
})
|
||||
|
||||
It("returns not found for missing nodes", func() {
|
||||
Expect(request(DrainNodeEndpoint(registry), "missing").Code).To(Equal(http.StatusNotFound))
|
||||
Expect(request(ResumeNodeEndpoint(registry), "missing").Code).To(Equal(http.StatusNotFound))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetNodeModelsEndpoint", func() {
|
||||
It("returns revision and cleanup state without serialized model options", func() {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
import { test, expect } from './coverage-fixtures.js'
|
||||
|
||||
test.describe('Operate console on a narrow screen', () => {
|
||||
test('ignores the desktop collapsed preference', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 390, height: 800 })
|
||||
await page.addInitScript(() => localStorage.setItem('localai_console_rail_collapsed', 'true'))
|
||||
await page.goto('/app/operate')
|
||||
|
||||
const rail = page.locator('.console-rail')
|
||||
await expect(rail).toHaveCSS('width', '374px')
|
||||
await expect(rail.getByText('Operate', { exact: true })).toBeVisible()
|
||||
await expect(rail.getByRole('button', { name: 'Expand Operate navigation' })).toBeVisible()
|
||||
await expect(rail.locator('.console-rail-collapse')).toBeHidden()
|
||||
})
|
||||
|
||||
test('expanding the rail leaves the overview on screen', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 390, height: 800 })
|
||||
await page.goto('/app/operate')
|
||||
|
||||
@@ -21,4 +21,24 @@ test.describe('Installed model backend logs link', () => {
|
||||
|
||||
await expect(page).toHaveURL(/\/app\/backend-logs\//)
|
||||
})
|
||||
|
||||
test('arrow navigation announces the active action through the focused menu', async ({ page }) => {
|
||||
await page.goto('/app/models?view=installed')
|
||||
await page.locator('[data-testid="installed-models-rail-item"]').first().click()
|
||||
const trigger = page.locator('button.action-menu__trigger').first()
|
||||
await trigger.focus()
|
||||
await trigger.press('Enter')
|
||||
|
||||
const menu = page.getByRole('menu')
|
||||
await expect(menu).toBeFocused()
|
||||
const firstItem = menu.getByRole('menuitem').first()
|
||||
await expect(menu).toHaveAttribute('aria-activedescendant', await firstItem.getAttribute('id'))
|
||||
|
||||
await menu.press('ArrowDown')
|
||||
const secondItem = menu.getByRole('menuitem').nth(1)
|
||||
await expect(menu).toHaveAttribute('aria-activedescendant', await secondItem.getAttribute('id'))
|
||||
await expect(menu).toBeFocused()
|
||||
await expect(firstItem).toHaveAttribute('tabindex', '-1')
|
||||
await expect(secondItem).toHaveAttribute('tabindex', '-1')
|
||||
})
|
||||
})
|
||||
@@ -55,4 +55,26 @@ test.describe('Navigation', () => {
|
||||
await expect(rail.locator('a.nav-item[href="/app/fine-tune"]')).toBeVisible()
|
||||
await expect(rail.locator('a.nav-item[href="/app/face"]')).toBeVisible()
|
||||
})
|
||||
|
||||
test('desktop console rail collapses to accessible icons and persists globally', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 900 })
|
||||
await page.goto('/app/backends')
|
||||
|
||||
const rail = page.locator('.console-rail')
|
||||
const collapse = rail.getByRole('button', { name: 'Collapse Operate navigation' })
|
||||
await expect(collapse).toBeVisible()
|
||||
await collapse.click()
|
||||
await expect(rail).toHaveClass(/console-rail--collapsed/)
|
||||
await expect(rail).toHaveCSS('width', '60px')
|
||||
await expect(rail.getByRole('link', { name: 'Backends', exact: true })).toHaveClass(/active/)
|
||||
await expect(rail.getByRole('link', { name: 'Overview', exact: true })).toHaveAttribute('title', 'Overview')
|
||||
await expect.poll(() => page.evaluate(() => localStorage.getItem('localai_console_rail_collapsed'))).toBe('true')
|
||||
|
||||
await page.goto('/app/agents')
|
||||
const buildRail = page.locator('.console-rail')
|
||||
await expect(buildRail).toHaveClass(/console-rail--collapsed/)
|
||||
await expect(buildRail.getByRole('button', { name: 'Expand Build navigation' })).toBeVisible()
|
||||
await page.reload()
|
||||
await expect(page.locator('.console-rail')).toHaveClass(/console-rail--collapsed/)
|
||||
})
|
||||
})
|
||||
@@ -1,9 +1,9 @@
|
||||
import { test, expect } from './coverage-fixtures.js'
|
||||
|
||||
const ID = 'n1'
|
||||
async function mockNode(page) {
|
||||
async function mockNode(page, overrides = {}) {
|
||||
await page.route(`**/api/nodes/${ID}`, r => r.fulfill({ status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify({ id: ID, name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', total_vram: 24e9, available_vram: 12e9, max_replicas_per_model: 1, labels: { env: 'prod' } }) }))
|
||||
body: JSON.stringify({ id: ID, name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', total_vram: 24e9, available_vram: 12e9, total_disk: 100e9, available_disk: 40e9, cpu_logical_cores: 16, cpu_usage_percent: 25, cpu_load_1: 2.5, max_replicas_per_model: 1, labels: { env: 'prod' }, ...overrides }) }))
|
||||
await page.route(`**/api/nodes/${ID}/models`, r => r.fulfill({ status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify([{ node_id: ID, model_name: 'llama-3.3', state: 'loaded', in_flight: 0, replica_index: 0 }]) }))
|
||||
await page.route(`**/api/nodes/${ID}/backends`, r => r.fulfill({ status: 200, contentType: 'application/json',
|
||||
@@ -19,16 +19,64 @@ test.describe('Node detail page', () => {
|
||||
await expect(page.getByText('llama-3.3')).toBeVisible()
|
||||
await expect(page.getByText('llama-cpp')).toBeVisible()
|
||||
await expect(page.getByText('env=prod')).toBeVisible()
|
||||
await expect(page.getByText('25.0% of 16 cores')).toBeVisible()
|
||||
await expect(page.getByText('2.50 load (1m)')).toBeVisible()
|
||||
await expect(page.getByText('37.3 GB / 93.1 GB')).toBeVisible()
|
||||
})
|
||||
|
||||
test('is reachable by clicking a roster panel', async ({ page }) => {
|
||||
await page.route('**/api/nodes', r => r.fulfill({ status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify([{ id: ID, name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy' }]) }))
|
||||
await page.route('**/api/nodes/models', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.route('**/api/nodes/scheduling', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await mockNode(page)
|
||||
await page.goto('/app/nodes')
|
||||
await page.locator('.node-panel').filter({ hasText: 'alpha' }).getByText('alpha').click()
|
||||
await page.getByRole('button', { name: 'Inspect alpha' }).click()
|
||||
await page.getByRole('link', { name: 'Open full node details' }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/app/nodes/${ID}$`))
|
||||
})
|
||||
|
||||
for (const [status, action] of [['healthy', 'Drain'], ['draining', 'Resume'], ['unhealthy', null], ['offline', null], ['unknown', null]]) {
|
||||
test(`shows only the accepted lifecycle action for ${status} nodes`, async ({ page }) => {
|
||||
await mockNode(page, { status })
|
||||
await page.goto(`/app/nodes/${ID}`)
|
||||
await expect(page.locator('.page-title').first()).toBeVisible({ timeout: 15_000 })
|
||||
await expect(page.getByRole('button', { name: /Approve/ })).toHaveCount(0)
|
||||
await expect(page.getByRole('button', { name: /Drain/ })).toHaveCount(action === 'Drain' ? 1 : 0)
|
||||
await expect(page.getByRole('button', { name: /Resume/ })).toHaveCount(action === 'Resume' ? 1 : 0)
|
||||
await expect(page.locator('.page-header__meta .btn-danger')).toContainText('Remove')
|
||||
})
|
||||
}
|
||||
|
||||
test('approves a pending node and refreshes its lifecycle controls', async ({ page }) => {
|
||||
let status = 'pending'
|
||||
let approvalRequests = 0
|
||||
await page.route(`**/api/nodes/${ID}`, r => r.fulfill({ status: 200, contentType: 'application/json',
|
||||
body: JSON.stringify({ id: ID, name: 'alpha', node_type: 'backend', status, labels: {} }) }))
|
||||
await page.route(`**/api/nodes/${ID}/models`, r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.route(`**/api/nodes/${ID}/backends`, r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.route(`**/api/nodes/${ID}/approve`, async r => {
|
||||
approvalRequests += 1
|
||||
status = 'healthy'
|
||||
await r.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
})
|
||||
await page.goto(`/app/nodes/${ID}`)
|
||||
|
||||
await page.getByRole('button', { name: 'Approve' }).click()
|
||||
await expect.poll(() => approvalRequests).toBe(1)
|
||||
await expect(page.getByText('Node approved')).toBeVisible()
|
||||
await expect(page.getByRole('button', { name: 'Drain' })).toBeVisible()
|
||||
await expect(page.locator('.page-header__meta .btn-danger')).toContainText('Remove')
|
||||
})
|
||||
|
||||
test('renders valid totals with missing available capacity as No data', async ({ page }) => {
|
||||
await mockNode(page, {
|
||||
total_vram: 24e9, available_vram: undefined,
|
||||
total_ram: 32e9, available_ram: undefined,
|
||||
total_disk: 100e9, available_disk: undefined,
|
||||
})
|
||||
await page.goto(`/app/nodes/${ID}`)
|
||||
await expect(page.locator('.node-detail__metrics')).toContainText('VRAM')
|
||||
await expect(page.locator('.node-detail__metrics')).toContainText('RAM')
|
||||
await expect(page.locator('.node-detail__metrics')).toContainText('Models disk free')
|
||||
await expect(page.locator('.node-detail__metrics').getByText('No data')).toHaveCount(3)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,747 @@
|
||||
import { test, expect } from './coverage-fixtures.js'
|
||||
|
||||
const baseNodes = [
|
||||
{ id: 'n1', name: 'atlas', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', labels: { zone: 'east' }, total_vram: 100, available_vram: 40, total_ram: 200, available_ram: 100, total_disk: 1000, available_disk: 600, cpu_logical_cores: 8, cpu_usage_percent: 25, cpu_load_1: 1.5, model_count: 3, in_flight_count: 2, last_heartbeat: '2026-09-14T00:00:00Z' },
|
||||
{ id: 'n2', name: 'borealis', node_type: 'backend', address: '10.0.0.2:50051', status: 'pending', labels: { zone: 'west' }, total_vram: 100, available_vram: 10, total_ram: 200, available_ram: 10, total_disk: 1000, available_disk: 100, cpu_logical_cores: 16, cpu_usage_percent: 50, cpu_load_1: 4, model_count: 1, in_flight_count: 0 },
|
||||
{ id: 'n3', name: 'legacy', node_type: 'agent', address: '10.0.0.3:50051', status: 'offline', labels: {} },
|
||||
]
|
||||
|
||||
async function mockNodes(page, nodes = baseNodes) {
|
||||
await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) }))
|
||||
}
|
||||
|
||||
async function mockFullOperateNavigation(page) {
|
||||
await page.route('**/api/features', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ distributed: true }),
|
||||
}))
|
||||
await page.route('**/api/auth/status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
authEnabled: true,
|
||||
staticApiKeyRequired: false,
|
||||
providers: ['local'],
|
||||
user: { id: 'admin', name: 'Admin', role: 'admin', provider: 'local' },
|
||||
}),
|
||||
}))
|
||||
}
|
||||
|
||||
const baseModels = [
|
||||
{ id: 'r1', node_id: 'n1', model_name: 'Llama 3.2', replica_index: 0, address: '10.0.0.1:50101', state: 'loaded', in_flight: 2, backend_type: 'llama-cpp', last_used: '2026-09-14T10:00:00Z' },
|
||||
{ id: 'r2', node_id: 'n1', model_name: 'Llama 3.2', replica_index: 1, address: '10.0.0.1:50102', state: 'loaded', in_flight: 0, backend_type: 'llama-cpp', last_used: '2026-09-14T10:30:00Z' },
|
||||
{ id: 'r3', node_id: 'n2', model_name: 'Llama 3.2', replica_index: 0, address: '10.0.0.2:50101', state: 'loaded', in_flight: 1, backend_type: 'vllm', last_used: '2026-09-14T11:00:00Z' },
|
||||
{ id: 'r4', node_id: 'missing', model_name: 'Whisper large v3', replica_index: 0, address: '10.0.0.9:50101', state: 'loaded', in_flight: 0, backend_type: 'whisper', last_used: '2026-09-14T09:00:00Z' },
|
||||
]
|
||||
|
||||
test.describe('Nodes fleet dashboard', () => {
|
||||
test('uses the standard Operate navigation at a desktop viewport', async ({ page }) => {
|
||||
await mockFullOperateNavigation(page)
|
||||
await mockNodes(page, [baseNodes[0]])
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
const primaryOperate = page.locator('.sidebar-nav a.nav-item', { hasText: 'Operate' })
|
||||
await expect(primaryOperate).toBeVisible({ timeout: 15_000 })
|
||||
await expect(primaryOperate).toHaveClass(/active/)
|
||||
|
||||
const rail = page.locator('.console-layout > .console-rail')
|
||||
await expect(rail).toBeVisible()
|
||||
await expect(rail.locator('a.nav-item')).toHaveCount(13)
|
||||
await expect(rail.locator('a[href="/app/nodes"]')).toHaveClass(/active/)
|
||||
await expect(rail.locator('a[href$="/swagger/index.html"]')).toHaveAttribute('target', '_blank')
|
||||
})
|
||||
|
||||
test('uses the standard collapsible Operate rail on mobile', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
await mockFullOperateNavigation(page)
|
||||
await mockNodes(page, [baseNodes[0]])
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
await page.getByRole('button', { name: 'Open menu' }).click()
|
||||
await expect(page.locator('.sidebar-nav a.nav-item', { hasText: 'Operate' })).toBeVisible()
|
||||
await page.getByRole('button', { name: 'Close menu' }).click()
|
||||
|
||||
const rail = page.locator('.console-layout > .console-rail')
|
||||
await expect(rail).toBeVisible()
|
||||
await expect(rail.locator('.console-rail-groups')).toBeHidden()
|
||||
await rail.getByRole('button', { name: 'Expand Operate navigation' }).click()
|
||||
await expect(rail.locator('.console-rail-groups')).toBeVisible()
|
||||
await expect(rail.locator('a.nav-item')).toHaveCount(13)
|
||||
})
|
||||
|
||||
test('shows aggregate health, capacity, attention filtering, search, sorting, and grouping', async ({ page }) => {
|
||||
await mockNodes(page)
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.getByLabel('Fleet health summary')).toContainText('3 nodes', { timeout: 15_000 })
|
||||
await expect(page.getByLabel('VRAM capacity')).toContainText('150 B / 200 B')
|
||||
await expect(page.getByLabel('CPU capacity')).toContainText('10 busy / 24 cores')
|
||||
await expect(page.getByLabel('Models disk capacity')).toContainText('1.3 KB / 2 KB')
|
||||
await expect(page.getByRole('button', { name: /Needs attention.*2/ })).toBeVisible()
|
||||
await page.getByRole('button', { name: /Low VRAM/ }).click()
|
||||
await expect(page.getByRole('row', { name: /borealis/ })).toBeVisible()
|
||||
await expect(page.getByRole('row', { name: /atlas/ })).toHaveCount(0)
|
||||
await page.getByRole('button', { name: /Low VRAM/ }).click()
|
||||
await page.getByRole('searchbox', { name: 'Search nodes' }).fill('legacy')
|
||||
await expect(page.getByRole('row', { name: /legacy/ })).toBeVisible()
|
||||
await page.getByRole('searchbox', { name: 'Search nodes' }).fill('')
|
||||
await page.getByRole('button', { name: /Sort by node/ }).click()
|
||||
await expect(page.locator('tbody tr').first()).toContainText('legacy')
|
||||
await page.getByLabel('Group nodes').selectOption('label:zone')
|
||||
await expect(page.getByText('Unlabelled', { exact: true })).toBeVisible()
|
||||
await expect(page.getByText('east', { exact: true })).toBeVisible()
|
||||
})
|
||||
|
||||
test('mounts only 50 rows and clamps pagination for a 1,000-node fleet', async ({ page }) => {
|
||||
const nodes = Array.from({ length: 1000 }, (_, index) => ({ id: `node-${index}`, name: `worker-${String(index).padStart(4, '0')}`, node_type: 'backend', address: `10.0.${Math.floor(index / 255)}.${index % 255}:50051`, status: 'healthy' }))
|
||||
await mockNodes(page, nodes)
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.locator('tbody tr')).toHaveCount(50, { timeout: 15_000 })
|
||||
await expect(page.getByText('Page 1 of 20')).toBeVisible()
|
||||
await page.getByRole('button', { name: 'Next page' }).click()
|
||||
await expect(page.getByText('Page 2 of 20')).toBeVisible()
|
||||
await page.getByRole('searchbox', { name: 'Search nodes' }).fill('worker-0000')
|
||||
await expect(page.getByText('Page 1 of 1')).toBeVisible()
|
||||
})
|
||||
|
||||
test('filters bulk actions by lifecycle state, reports skipped nodes, and prevents overlapping batches', async ({ page }) => {
|
||||
const nodes = Array.from({ length: 12 }, (_, index) => ({
|
||||
id: `n${index}`,
|
||||
name: `worker-${index}`,
|
||||
node_type: 'backend',
|
||||
status: index === 9 ? 'pending' : index === 10 ? 'draining' : index === 11 ? 'offline' : 'healthy',
|
||||
}))
|
||||
await mockNodes(page, nodes)
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const drainRequests = []
|
||||
const resumeRequests = []
|
||||
await page.route('**/api/nodes/*/drain', async route => {
|
||||
active += 1
|
||||
peak = Math.max(peak, active)
|
||||
await new Promise(resolve => setTimeout(resolve, 30))
|
||||
active -= 1
|
||||
const id = route.request().url().split('/').at(-2)
|
||||
drainRequests.push(id)
|
||||
await route.fulfill({ status: id === 'n8' ? 500 : 200, contentType: 'application/json', body: id === 'n8' ? '{"error":"failed"}' : '{}' })
|
||||
})
|
||||
await page.route('**/api/nodes/*/resume', async route => {
|
||||
const id = route.request().url().split('/').at(-2)
|
||||
resumeRequests.push(id)
|
||||
await route.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('checkbox', { name: 'Select visible nodes' }).check()
|
||||
await page.getByRole('searchbox', { name: 'Search nodes' }).fill('worker-1')
|
||||
await expect(page.getByText('12 selected')).toBeVisible()
|
||||
await page.getByRole('button', { name: 'Drain selected' }).evaluate(button => {
|
||||
button.click()
|
||||
button.click()
|
||||
})
|
||||
await expect.poll(() => drainRequests.length).toBe(9)
|
||||
expect(peak).toBeLessThanOrEqual(8)
|
||||
expect(drainRequests.sort()).toEqual(Array.from({ length: 9 }, (_, index) => `n${index}`).sort())
|
||||
await expect(page.getByText(/8 succeeded, 1 failed, 3 skipped/)).toBeVisible()
|
||||
|
||||
await page.getByRole('button', { name: 'Resume selected' }).click()
|
||||
await expect.poll(() => resumeRequests).toEqual(['n10'])
|
||||
await expect(page.getByText(/1 succeeded, 0 failed, 11 skipped/)).toBeVisible()
|
||||
expect(drainRequests).not.toContain('n9')
|
||||
expect(resumeRequests).not.toContain('n9')
|
||||
})
|
||||
|
||||
test('disables bulk controls and remove confirmation while removal is running', async ({ page }) => {
|
||||
await mockNodes(page, [baseNodes[0]])
|
||||
let finishRemove
|
||||
await page.route('**/api/nodes/n1', async route => {
|
||||
if (route.request().method() !== 'DELETE') return route.fallback()
|
||||
await new Promise(resolve => { finishRemove = resolve })
|
||||
await route.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('checkbox', { name: 'Select atlas' }).check()
|
||||
await page.getByRole('button', { name: 'Remove selected' }).click()
|
||||
await page.getByRole('button', { name: 'Remove nodes' }).click()
|
||||
|
||||
await expect(page.getByRole('button', { name: 'Removing…' })).toBeDisabled()
|
||||
await expect(page.getByRole('button', { name: 'Cancel' })).toBeDisabled()
|
||||
finishRemove()
|
||||
await expect(page.getByText(/1 succeeded, 0 failed, 0 skipped/)).toBeVisible()
|
||||
})
|
||||
|
||||
test('fetches backends only when an inspector opens and shows unknown legacy metrics', async ({ page }) => {
|
||||
await mockNodes(page)
|
||||
let backendRequests = 0
|
||||
await page.route('**/api/nodes/n3/backends', route => {
|
||||
backendRequests += 1
|
||||
return route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"tool-runner"}]' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
expect(backendRequests).toBe(0)
|
||||
await page.getByRole('button', { name: 'Inspect legacy' }).click()
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await expect(inspector).toContainText('legacy')
|
||||
await expect(inspector).toContainText('No data')
|
||||
await expect(inspector).toContainText('1 backend')
|
||||
expect(backendRequests).toBe(1)
|
||||
await expect(page.getByRole('link', { name: 'Open full node details' })).toHaveAttribute('href', '/app/nodes/n3')
|
||||
})
|
||||
|
||||
test('keeps pending approval visible', async ({ page }) => {
|
||||
await mockNodes(page, [baseNodes[1]])
|
||||
await page.route('**/api/nodes/n2/approve', route => route.fulfill({ status: 200, contentType: 'application/json', body: '{}' }))
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.getByRole('button', { name: 'Approve borealis' })).toBeVisible({ timeout: 15_000 })
|
||||
})
|
||||
|
||||
test('shows inspector lifecycle controls only for server-accepted states', async ({ page }) => {
|
||||
const statuses = ['healthy', 'draining', 'pending', 'unhealthy', 'offline', 'unknown']
|
||||
await mockNodes(page, statuses.map((status, index) => ({
|
||||
id: `state-${index}`,
|
||||
name: `node-${status}`,
|
||||
node_type: 'backend',
|
||||
status,
|
||||
})))
|
||||
await page.route('**/api/nodes/*/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
for (const status of statuses) {
|
||||
await page.getByRole('button', { name: `Inspect node-${status}` }).click()
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await expect(inspector.getByRole('button', { name: 'Approve', exact: true })).toHaveCount(status === 'pending' ? 1 : 0)
|
||||
await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toHaveCount(status === 'healthy' ? 1 : 0)
|
||||
await expect(inspector.getByRole('button', { name: 'Resume', exact: true })).toHaveCount(status === 'draining' ? 1 : 0)
|
||||
await inspector.getByRole('button', { name: 'Close node inspector' }).click()
|
||||
}
|
||||
})
|
||||
|
||||
test('approves a pending node from the model-to-node drilldown', async ({ page }) => {
|
||||
let status = 'pending'
|
||||
let approvalRequests = 0
|
||||
await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([
|
||||
{ id: 'pending-node', name: 'pending-worker', node_type: 'backend', status },
|
||||
]) }))
|
||||
await page.route('**/api/nodes/models', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([
|
||||
{ id: 'replica', node_id: 'pending-node', model_name: 'Pending model', replica_index: 0, state: 'loaded' },
|
||||
]) }))
|
||||
await page.route('**/api/nodes/pending-node/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.route('**/api/nodes/pending-node/approve', async route => {
|
||||
approvalRequests += 1
|
||||
status = 'healthy'
|
||||
await route.fulfill({ status: 200, contentType: 'application/json', body: '{}' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('tab', { name: 'Running models' }).click()
|
||||
await page.getByRole('button', { name: 'Inspect Pending model' }).click()
|
||||
await page.getByRole('button', { name: 'Open node pending-worker' }).click()
|
||||
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await inspector.getByRole('button', { name: 'Approve', exact: true }).click()
|
||||
await expect.poll(() => approvalRequests).toBe(1)
|
||||
await expect(page.getByText('Node approved')).toBeVisible()
|
||||
await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toBeVisible()
|
||||
})
|
||||
|
||||
test('keeps incomplete capacity unknown throughout the fleet view', async ({ page }) => {
|
||||
await mockNodes(page, [{
|
||||
id: 'incomplete', name: 'incomplete-capacity', node_type: 'backend', status: 'healthy',
|
||||
total_vram: 100, total_ram: 200, total_disk: 300,
|
||||
}])
|
||||
await page.route('**/api/nodes/incomplete/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
await expect(page.getByLabel('VRAM capacity')).toContainText('No data')
|
||||
await expect(page.getByLabel('VRAM capacity')).toContainText('1 node unavailable')
|
||||
await expect(page.getByRole('button', { name: /Low VRAM.*0/ })).toBeVisible()
|
||||
const row = page.getByRole('row', { name: /incomplete-capacity/ })
|
||||
await expect(row.getByText('No data')).toHaveCount(3)
|
||||
await page.getByRole('button', { name: 'Inspect incomplete-capacity' }).click()
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await expect(inspector.getByText('No data')).toHaveCount(4)
|
||||
})
|
||||
|
||||
test('announces complete and partial capacity coverage without adding visible clutter', async ({ page }) => {
|
||||
const partialNode = {
|
||||
...baseNodes[0],
|
||||
id: 'n-partial',
|
||||
name: 'partial-capacity',
|
||||
total_vram: 100,
|
||||
available_vram: 50,
|
||||
total_ram: undefined,
|
||||
available_ram: undefined,
|
||||
}
|
||||
await mockNodes(page, [baseNodes[0], partialNode])
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
const vram = page.getByLabel('VRAM capacity', { exact: true })
|
||||
const ram = page.getByLabel('RAM capacity', { exact: true })
|
||||
await expect(vram).toContainText('Capacity coverage: 2 of 2 nodes reporting; 0 unknown.', { timeout: 15_000 })
|
||||
await expect(ram).toContainText('Capacity coverage: 1 of 2 nodes reporting; 1 unknown.')
|
||||
await expect(vram.locator('.fleet-gauge__coverage')).toHaveCount(0)
|
||||
await expect(ram.locator('.fleet-gauge__coverage')).toHaveText('1 node unavailable')
|
||||
})
|
||||
|
||||
test('keeps checkbox keyboard activation from opening the inspector', async ({ page }) => {
|
||||
await mockNodes(page, [baseNodes[0]])
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
const checkbox = page.getByRole('checkbox', { name: 'Select atlas' })
|
||||
await checkbox.focus()
|
||||
await checkbox.press('Space')
|
||||
|
||||
await expect(checkbox).toBeChecked()
|
||||
await expect(page.getByRole('complementary', { name: 'Node inspector' })).toHaveCount(0)
|
||||
})
|
||||
|
||||
test('keeps the low-density composition while inspecting at a desktop viewport', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1050 })
|
||||
await mockNodes(page)
|
||||
await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"llama-cpp"},{"name":"whisper"}]' }))
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
const overview = page.getByRole('region', { name: 'Fleet overview' })
|
||||
const workbench = page.getByRole('region', { name: 'Fleet workbench' })
|
||||
await expect(overview).toBeVisible({ timeout: 15_000 })
|
||||
await expect(workbench).toBeVisible()
|
||||
await expect(page.locator('.console-layout > .console-rail')).toBeVisible()
|
||||
await expect(page.locator('.fleet-select-wrap')).toHaveCount(3)
|
||||
await expect(page.getByLabel('Filter status')).toHaveCSS('appearance', 'none')
|
||||
await expect(page.locator('.fleet-bulkbar')).toHaveCount(0)
|
||||
|
||||
const overviewBefore = await overview.boundingBox()
|
||||
const fleetBefore = await page.locator('#fleet-nodes-panel').boundingBox()
|
||||
const cells = overview.locator('.fleet-overview__cell')
|
||||
await expect(cells).toHaveCount(5)
|
||||
const cellTops = await cells.evaluateAll(items => items.map(cell => Math.round(cell.getBoundingClientRect().top)))
|
||||
expect(new Set(cellTops).size).toBe(1)
|
||||
const attention = page.getByRole('complementary', { name: 'Attention queue' })
|
||||
await expect(attention).toBeVisible()
|
||||
const overviewBox = await overview.boundingBox()
|
||||
const attentionBox = await attention.boundingBox()
|
||||
expect(attentionBox.y).toBeGreaterThanOrEqual(overviewBox.y + overviewBox.height)
|
||||
|
||||
const checkbox = page.getByRole('checkbox', { name: 'Select atlas' })
|
||||
await checkbox.check()
|
||||
await expect(page.getByRole('row', { name: /atlas/ })).toHaveClass(/is-selected/)
|
||||
await expect(page.locator('.fleet-bulkbar')).toBeVisible()
|
||||
await expect(page.getByRole('button', { name: 'Clear selection' })).toBeVisible()
|
||||
|
||||
const inspectNode = page.getByRole('button', { name: 'Inspect atlas' })
|
||||
await inspectNode.focus()
|
||||
await inspectNode.press('Enter')
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await expect(inspector).toBeVisible()
|
||||
await expect(inspector.getByRole('heading', { name: 'Node' })).toBeVisible()
|
||||
await expect(inspector.getByRole('heading', { name: 'Resources' })).toBeVisible()
|
||||
await expect(inspector.getByRole('heading', { name: 'Workload' })).toBeVisible()
|
||||
await expect(inspector.locator('.node-inspector__resource')).toHaveCount(2)
|
||||
|
||||
const overviewAfter = await overview.boundingBox()
|
||||
const fleetAfter = await page.locator('#fleet-nodes-panel').boundingBox()
|
||||
expect(Math.abs(overviewAfter.width - overviewBefore.width)).toBeLessThanOrEqual(1)
|
||||
expect(Math.abs(fleetBefore.width - fleetAfter.width)).toBeLessThanOrEqual(1)
|
||||
await expect(inspector).toHaveCSS('position', 'fixed')
|
||||
await expect.poll(async () => {
|
||||
const inspectorBox = await inspector.boundingBox()
|
||||
return Math.max(
|
||||
Math.abs(inspectorBox.x + inspectorBox.width - 1584),
|
||||
Math.abs(inspectorBox.y - 16),
|
||||
Math.abs(inspectorBox.height - 1018),
|
||||
)
|
||||
}).toBeLessThanOrEqual(1)
|
||||
})
|
||||
|
||||
test('reflows the overview and presents a contained drawer at a narrow viewport', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 640, height: 900 })
|
||||
await mockNodes(page, [baseNodes[0]])
|
||||
await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
const overview = page.getByRole('region', { name: 'Fleet overview' })
|
||||
await expect(overview).toBeVisible({ timeout: 15_000 })
|
||||
await expect(page.locator('.console-layout > .console-rail')).toBeVisible()
|
||||
const overviewBox = await overview.boundingBox()
|
||||
expect(overviewBox.width).toBeGreaterThan(500)
|
||||
const cells = overview.locator('.fleet-overview__cell')
|
||||
const tops = await cells.evaluateAll(items => items.map(item => Math.round(item.getBoundingClientRect().top)))
|
||||
expect(new Set(tops).size).toBeGreaterThan(1)
|
||||
await expect(overview.getByLabel('Fleet health summary')).toHaveCSS('grid-column-start', '1')
|
||||
await expect(overview.getByLabel('Fleet health summary')).toHaveCSS('grid-column-end', '-1')
|
||||
|
||||
const narrowInspectNode = page.getByRole('button', { name: 'Inspect atlas' })
|
||||
await narrowInspectNode.focus()
|
||||
await narrowInspectNode.press('Enter')
|
||||
const inspector = page.getByRole('dialog', { name: 'Node inspector' })
|
||||
await expect(inspector).toBeVisible()
|
||||
await expect(inspector).toHaveAttribute('aria-modal', 'true')
|
||||
await expect(inspector).toHaveCSS('position', 'fixed')
|
||||
await expect(page.locator('.node-inspector__scrim')).toBeVisible()
|
||||
await expect(page.locator('body')).toHaveCSS('overflow', 'hidden')
|
||||
await expect(page.getByRole('region', { name: 'Fleet workbench', includeHidden: true })).toHaveAttribute('inert', '')
|
||||
await expect(page.getByRole('region', { name: 'Fleet workbench', includeHidden: true })).toHaveAttribute('aria-hidden', 'true')
|
||||
const workbenchBox = await page.locator('.fleet-workbench').boundingBox()
|
||||
expect(workbenchBox.width).toBeLessThanOrEqual(600)
|
||||
await expect.poll(async () => (await inspector.boundingBox()).y).toBeLessThanOrEqual(1)
|
||||
const inspectorBox = await inspector.boundingBox()
|
||||
expect(inspectorBox.height).toBe(900)
|
||||
const close = inspector.getByRole('button', { name: 'Close node inspector' })
|
||||
await expect(close).toBeFocused()
|
||||
await close.press('Shift+Tab')
|
||||
await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toBeFocused()
|
||||
await page.keyboard.press('Tab')
|
||||
await expect(close).toBeFocused()
|
||||
await page.keyboard.press('Escape')
|
||||
await expect(inspector).toHaveCount(0)
|
||||
await expect(page.locator('body')).not.toHaveCSS('overflow', 'hidden')
|
||||
await expect(page.getByRole('region', { name: 'Fleet workbench' })).not.toHaveAttribute('inert', '')
|
||||
await expect(page.getByRole('region', { name: 'Fleet workbench' })).not.toHaveAttribute('aria-hidden', 'true')
|
||||
await expect(narrowInspectNode).toBeFocused()
|
||||
})
|
||||
|
||||
test('loads running models once on activation and drills model to node and back', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1050 })
|
||||
await mockNodes(page, baseNodes.map(node => node.id === 'n2' ? { ...node, status: 'healthy' } : node))
|
||||
let modelRequests = 0
|
||||
let backendRequests = 0
|
||||
await page.route('**/api/nodes/models', route => {
|
||||
modelRequests += 1
|
||||
return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(baseModels) })
|
||||
})
|
||||
await page.route('**/api/nodes/n1/backends', route => {
|
||||
backendRequests += 1
|
||||
return route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"llama-cpp"}]' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.getByRole('table', { name: 'Fleet nodes' })).toBeVisible({ timeout: 15_000 })
|
||||
expect(modelRequests).toBe(0)
|
||||
expect(backendRequests).toBe(0)
|
||||
|
||||
const nodesTab = page.getByRole('tab', { name: 'Nodes' })
|
||||
const modelsTab = page.getByRole('tab', { name: 'Running models' })
|
||||
const nodesPanel = page.locator('#fleet-nodes-panel')
|
||||
const modelsPanel = page.locator('#fleet-models-panel')
|
||||
await expect(nodesTab).toHaveAttribute('aria-controls', 'fleet-nodes-panel')
|
||||
await expect(nodesTab).toHaveAttribute('tabindex', '0')
|
||||
await expect(modelsTab).toHaveAttribute('aria-controls', 'fleet-models-panel')
|
||||
await expect(modelsTab).toHaveAttribute('tabindex', '-1')
|
||||
await expect(nodesPanel).toHaveAttribute('role', 'tabpanel')
|
||||
await expect(nodesPanel).toHaveAttribute('aria-labelledby', 'fleet-nodes-tab')
|
||||
await expect(nodesPanel).not.toHaveAttribute('hidden', '')
|
||||
await expect(nodesPanel).toBeVisible()
|
||||
await expect(modelsPanel).toHaveAttribute('role', 'tabpanel')
|
||||
await expect(modelsPanel).toHaveAttribute('aria-labelledby', 'fleet-models-tab')
|
||||
await expect(modelsPanel).toHaveAttribute('hidden', '')
|
||||
await expect(modelsPanel).toBeHidden()
|
||||
await expect(modelsPanel.getByRole('table', { name: 'Running models' })).toHaveCount(0)
|
||||
await expect(nodesPanel.locator('tbody tr')).toHaveCount(3)
|
||||
|
||||
await nodesTab.focus()
|
||||
await nodesTab.press('ArrowRight')
|
||||
await expect(modelsTab).toBeFocused()
|
||||
await expect(modelsTab).toHaveAttribute('aria-selected', 'true')
|
||||
await expect(modelsTab).toHaveAttribute('tabindex', '0')
|
||||
await expect(nodesTab).toHaveAttribute('tabindex', '-1')
|
||||
await expect(nodesPanel).toHaveAttribute('hidden', '')
|
||||
await expect(nodesPanel).toBeHidden()
|
||||
await expect(modelsPanel).not.toHaveAttribute('hidden', '')
|
||||
await expect(modelsPanel).toBeVisible()
|
||||
await expect(page.getByText('Current loaded replicas on healthy nodes')).toBeVisible()
|
||||
await expect(page.getByRole('table', { name: 'Running models' })).toBeVisible()
|
||||
await expect(page.getByRole('row', { name: /Llama 3.2/ })).toContainText('3')
|
||||
expect(modelRequests).toBe(1)
|
||||
expect(backendRequests).toBe(0)
|
||||
|
||||
const modelControl = page.getByRole('button', { name: 'Inspect Llama 3.2' })
|
||||
await expect(modelControl).not.toHaveAttribute('aria-selected')
|
||||
await expect(modelControl).toHaveAttribute('aria-pressed', 'false')
|
||||
await expect(modelControl).toHaveAttribute('aria-expanded', 'false')
|
||||
await expect(modelControl).not.toHaveAttribute('aria-controls')
|
||||
await modelControl.focus()
|
||||
await modelControl.press('Enter')
|
||||
const modelInspector = page.getByRole('complementary', { name: 'Model inspector' })
|
||||
const closeModel = page.getByRole('button', { name: 'Close model inspector' })
|
||||
await expect(closeModel).toBeFocused()
|
||||
await expect(modelControl).not.toHaveAttribute('aria-selected')
|
||||
await expect(modelControl).toHaveAttribute('aria-pressed', 'true')
|
||||
await expect(modelControl).toHaveAttribute('aria-expanded', 'true')
|
||||
await expect(modelControl).toHaveAttribute('aria-current', 'true')
|
||||
await expect(modelControl).toHaveAttribute('aria-controls', 'model-inspector')
|
||||
await expect(modelInspector).toContainText('2 replicas')
|
||||
await expect(modelInspector).toContainText('borealis')
|
||||
const atlasControl = modelInspector.getByRole('button', { name: /Open node atlas/ })
|
||||
await atlasControl.focus()
|
||||
await atlasControl.press('Enter')
|
||||
await expect(page.getByRole('complementary', { name: 'Node inspector' })).toBeVisible()
|
||||
await expect(modelControl).toHaveAttribute('aria-pressed', 'true')
|
||||
await expect(modelControl).toHaveAttribute('aria-expanded', 'false')
|
||||
await expect(modelControl).not.toHaveAttribute('aria-controls')
|
||||
const backToModel = page.getByRole('button', { name: 'Back to Llama 3.2' })
|
||||
await expect(backToModel).toBeFocused()
|
||||
await expect.poll(() => backendRequests).toBe(1)
|
||||
expect(modelRequests).toBe(1)
|
||||
await backToModel.press('Enter')
|
||||
await expect(page.getByRole('complementary', { name: 'Model inspector' })).toBeVisible()
|
||||
await expect(page.getByRole('button', { name: /Open node atlas/ })).toBeFocused()
|
||||
await closeModel.click()
|
||||
await expect(page.getByRole('complementary', { name: 'Model inspector' })).toHaveCount(0)
|
||||
await expect(modelControl).toBeFocused()
|
||||
await expect(modelControl).not.toHaveAttribute('aria-selected')
|
||||
await expect(modelControl).toHaveAttribute('aria-pressed', 'false')
|
||||
await expect(modelControl).toHaveAttribute('aria-expanded', 'false')
|
||||
await expect(modelControl).not.toHaveAttribute('aria-controls')
|
||||
|
||||
await page.getByRole('button', { name: 'Inspect Whisper large v3' }).click()
|
||||
await expect(page.getByRole('complementary', { name: 'Model inspector' })).toContainText('missing')
|
||||
await expect(page.getByRole('complementary', { name: 'Model inspector' })).toContainText('Unknown')
|
||||
await page.getByRole('button', { name: 'Close model inspector' }).click()
|
||||
|
||||
await modelsTab.focus()
|
||||
await modelsTab.press('ArrowLeft')
|
||||
await expect(nodesTab).toBeFocused()
|
||||
await expect(nodesTab).toHaveAttribute('aria-selected', 'true')
|
||||
await expect(nodesPanel).not.toHaveAttribute('hidden', '')
|
||||
await expect(modelsPanel).toHaveAttribute('hidden', '')
|
||||
await expect(modelsPanel.locator('tbody tr')).toHaveCount(2)
|
||||
await nodesTab.press('ArrowRight')
|
||||
await expect(modelsPanel.locator('tbody tr')).toHaveCount(2)
|
||||
expect(modelRequests).toBe(1)
|
||||
})
|
||||
|
||||
test('treats the model inspector as a modal drawer on mobile', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
await mockNodes(page)
|
||||
await page.route('**/api/nodes/models', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(baseModels) }))
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('tab', { name: 'Running models' }).click()
|
||||
await page.getByRole('button', { name: 'Inspect Llama 3.2' }).click()
|
||||
|
||||
const inspector = page.getByRole('dialog', { name: 'Model inspector' })
|
||||
await expect(inspector).toHaveAttribute('aria-modal', 'true')
|
||||
await expect(inspector.getByRole('button', { name: 'Close model inspector' })).toBeFocused()
|
||||
await expect(page.locator('.fleet-workbench')).toHaveAttribute('inert', '')
|
||||
await inspector.getByRole('button', { name: 'Close model inspector' }).press('Shift+Tab')
|
||||
await expect(inspector.getByRole('button', { name: 'Close', exact: true })).toBeFocused()
|
||||
await page.keyboard.press('Tab')
|
||||
await expect(inspector.getByRole('button', { name: 'Close model inspector' })).toBeFocused()
|
||||
})
|
||||
|
||||
test('keeps the node inspector open when Escape dismisses its confirmation dialog', async ({ page }) => {
|
||||
await mockNodes(page)
|
||||
await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('checkbox', { name: 'Select atlas' }).check()
|
||||
await page.getByRole('button', { name: 'Inspect atlas' }).click()
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await expect(inspector).toBeVisible()
|
||||
|
||||
await page.getByRole('button', { name: 'Remove selected' }).click()
|
||||
await expect(page.getByRole('alertdialog')).toBeVisible()
|
||||
await page.keyboard.press('Escape')
|
||||
|
||||
await expect(page.getByRole('alertdialog')).toHaveCount(0)
|
||||
await expect(inspector).toBeVisible()
|
||||
})
|
||||
|
||||
test('stops a running model once from an accessible row menu and refreshes inventory', async ({ page }) => {
|
||||
await mockNodes(page)
|
||||
let modelRequests = 0
|
||||
let stopRequests = 0
|
||||
let stopBody
|
||||
let finishStop
|
||||
await page.route('**/api/nodes/models', route => {
|
||||
modelRequests += 1
|
||||
const rows = modelRequests === 1 ? baseModels : baseModels.filter(row => row.model_name !== 'Llama 3.2')
|
||||
return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(rows) })
|
||||
})
|
||||
await page.route('**/backend/shutdown', async route => {
|
||||
stopRequests += 1
|
||||
stopBody = route.request().postDataJSON()
|
||||
await new Promise(resolve => { finishStop = resolve })
|
||||
await route.fulfill({ status: 200, contentType: 'application/json', body: '{"message":"ok"}' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
const modelsTab = page.getByRole('tab', { name: 'Running models' })
|
||||
await modelsTab.click()
|
||||
|
||||
const trigger = page.getByRole('button', { name: 'Actions for Llama 3.2' })
|
||||
await expect(trigger).toBeVisible()
|
||||
await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))))
|
||||
await expect(modelsTab).toBeFocused()
|
||||
|
||||
await trigger.focus()
|
||||
await trigger.press('Enter')
|
||||
const menu = page.getByRole('menu', { name: 'Llama 3.2 actions' })
|
||||
await expect(menu).toBeVisible()
|
||||
await expect(menu).toBeFocused()
|
||||
await menu.press('Escape')
|
||||
await expect(menu).toHaveCount(0)
|
||||
await expect(trigger).toBeFocused()
|
||||
await expect(page.getByRole('complementary', { name: 'Model inspector' })).toHaveCount(0)
|
||||
|
||||
await trigger.click()
|
||||
await page.locator('.model-workbench__scope').click()
|
||||
await expect(menu).toHaveCount(0)
|
||||
await expect(page.getByRole('complementary', { name: 'Model inspector' })).toHaveCount(0)
|
||||
|
||||
await trigger.click()
|
||||
await menu.getByRole('menuitem', { name: 'Stop model…' }).click()
|
||||
const dialog = page.getByRole('alertdialog')
|
||||
await expect(dialog).toContainText('Stop Llama 3.2?')
|
||||
await expect(dialog).toContainText('Llama 3.2 has 3 loaded replicas across 2 unique nodes. This will stop all loaded placements on those nodes.')
|
||||
await expect(dialog.getByRole('button', { name: 'Stop model' })).toBeFocused()
|
||||
await page.keyboard.press('Tab')
|
||||
await expect(dialog.getByRole('button', { name: 'Cancel' })).toBeFocused()
|
||||
await page.keyboard.press('Shift+Tab')
|
||||
await expect(dialog.getByRole('button', { name: 'Stop model' })).toBeFocused()
|
||||
await dialog.getByRole('button', { name: 'Cancel' }).click()
|
||||
await expect(dialog).toHaveCount(0)
|
||||
await expect(trigger).toBeFocused()
|
||||
|
||||
await trigger.click()
|
||||
await menu.getByRole('menuitem', { name: 'Stop model…' }).click()
|
||||
await dialog.getByRole('button', { name: 'Stop model' }).evaluate(button => {
|
||||
button.click()
|
||||
button.click()
|
||||
})
|
||||
|
||||
await expect(dialog.getByRole('button', { name: 'Stopping…' })).toBeDisabled()
|
||||
await expect(dialog.getByRole('button', { name: 'Cancel' })).toBeDisabled()
|
||||
await expect.poll(() => stopRequests).toBe(1)
|
||||
expect(stopBody).toEqual({ model: 'Llama 3.2' })
|
||||
finishStop()
|
||||
await expect.poll(() => modelRequests).toBe(2)
|
||||
await expect(page.getByText('Stopped Llama 3.2: 3 replicas across 2 nodes.')).toBeVisible()
|
||||
await expect(page.getByRole('button', { name: 'Inspect Llama 3.2' })).toHaveCount(0)
|
||||
})
|
||||
|
||||
test('refreshes model inventory and warns about partial shutdown after a stop failure', async ({ page }) => {
|
||||
await mockNodes(page)
|
||||
let modelRequests = 0
|
||||
let stopRequests = 0
|
||||
await page.route('**/api/nodes/models', route => {
|
||||
modelRequests += 1
|
||||
return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(baseModels) })
|
||||
})
|
||||
await page.route('**/backend/shutdown', route => {
|
||||
stopRequests += 1
|
||||
return route.fulfill({ status: 500, contentType: 'application/json', body: '{"error":"controller timed out"}' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('tab', { name: 'Running models' }).click()
|
||||
await page.getByRole('button', { name: 'Actions for Whisper large v3' }).click()
|
||||
await page.getByRole('menuitem', { name: 'Stop model…' }).click()
|
||||
await page.getByRole('alertdialog').getByRole('button', { name: 'Stop model' }).click()
|
||||
|
||||
await expect.poll(() => stopRequests).toBe(1)
|
||||
await expect.poll(() => modelRequests).toBe(2)
|
||||
await expect(page.getByText(/Could not stop Whisper large v3:.*Some replicas may already have stopped\./)).toBeVisible()
|
||||
await expect(page.getByRole('button', { name: 'Inspect Whisper large v3' })).toBeVisible()
|
||||
})
|
||||
|
||||
test('moves focus into and restores it from the node inspector', async ({ page }) => {
|
||||
await mockNodes(page, [baseNodes[0]])
|
||||
await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
const nodeControl = page.getByRole('button', { name: 'Inspect atlas' })
|
||||
await nodeControl.click()
|
||||
const closeNode = page.getByRole('button', { name: 'Close node inspector' })
|
||||
await expect(closeNode).toBeFocused()
|
||||
await closeNode.click()
|
||||
await expect(nodeControl).toBeFocused()
|
||||
})
|
||||
|
||||
test('keeps drawer actions visible for a one-row filtered fleet', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 900 })
|
||||
await mockNodes(page, baseNodes)
|
||||
await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"llama-cpp"}]' }))
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('searchbox', { name: 'Search nodes' }).fill('atlas')
|
||||
await expect(page.getByRole('row', { name: /atlas/ })).toBeVisible()
|
||||
await page.getByRole('button', { name: 'Inspect atlas' }).click()
|
||||
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await expect(inspector.getByRole('heading', { name: 'Resources' })).toBeVisible()
|
||||
await expect(inspector.getByRole('heading', { name: 'Workload' })).toBeVisible()
|
||||
await expect(inspector.getByRole('link', { name: 'Open full node details' })).toBeVisible()
|
||||
await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toBeVisible()
|
||||
|
||||
const inspectorBox = await inspector.boundingBox()
|
||||
expect(Math.abs(inspectorBox.y - 16)).toBeLessThanOrEqual(1)
|
||||
expect(Math.abs(inspectorBox.height - 868)).toBeLessThanOrEqual(1)
|
||||
await expect(inspector.locator('.node-inspector__actions')).toHaveCSS('display', 'grid')
|
||||
})
|
||||
|
||||
test('keeps a desktop drawer in the visible viewport after opening from a long roster', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 800 })
|
||||
const nodes = Array.from({ length: 50 }, (_, index) => ({
|
||||
id: `long-${index}`,
|
||||
name: `long-worker-${String(index).padStart(2, '0')}`,
|
||||
node_type: 'backend',
|
||||
status: 'healthy',
|
||||
}))
|
||||
await mockNodes(page, nodes)
|
||||
await page.route('**/api/nodes/long-49/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.goto('/app/nodes')
|
||||
|
||||
await page.getByRole('button', { name: 'Inspect long-worker-49' }).click()
|
||||
expect(await page.evaluate(() => window.scrollY)).toBeGreaterThan(1000)
|
||||
const inspector = page.getByRole('complementary', { name: 'Node inspector' })
|
||||
await expect(inspector).toHaveCSS('position', 'fixed')
|
||||
await expect(inspector.getByRole('heading', { name: 'long-worker-49' })).toBeVisible()
|
||||
await expect(inspector.getByRole('link', { name: 'Open full node details' })).toBeVisible()
|
||||
const box = await inspector.boundingBox()
|
||||
expect(Math.abs(box.y - 16)).toBeLessThanOrEqual(1)
|
||||
expect(Math.abs(box.height - 768)).toBeLessThanOrEqual(1)
|
||||
})
|
||||
|
||||
test('shows model loading, error, retry, and empty states', async ({ page }) => {
|
||||
await mockNodes(page)
|
||||
let finishFirst
|
||||
let attempts = 0
|
||||
await page.route('**/api/nodes/models', async route => {
|
||||
attempts += 1
|
||||
if (attempts === 1) {
|
||||
await new Promise(resolve => { finishFirst = resolve })
|
||||
return route.fulfill({ status: 500, contentType: 'application/json', body: '{"error":"database unavailable"}' })
|
||||
}
|
||||
return route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })
|
||||
})
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('tab', { name: 'Running models' }).click()
|
||||
await expect(page.getByText('Loading running models…')).toBeVisible()
|
||||
finishFirst()
|
||||
await expect(page.getByText('Unable to load running models')).toBeVisible()
|
||||
await page.getByRole('button', { name: 'Retry loading running models' }).click()
|
||||
await expect(page.getByText('No running models')).toBeVisible()
|
||||
expect(attempts).toBe(2)
|
||||
})
|
||||
|
||||
test('mounts 50 of 1,000 running models and supports search and sorting', async ({ page }) => {
|
||||
await mockNodes(page)
|
||||
const models = Array.from({ length: 1000 }, (_, index) => ({
|
||||
id: `replica-${index}`,
|
||||
node_id: 'n1',
|
||||
model_name: `model-${String(index).padStart(4, '0')}`,
|
||||
replica_index: 0,
|
||||
address: `10.0.0.1:${51000 + index}`,
|
||||
state: 'loaded',
|
||||
in_flight: index % 7,
|
||||
backend_type: 'llama-cpp',
|
||||
last_used: new Date(Date.UTC(2026, 8, 1, 0, index)).toISOString(),
|
||||
}))
|
||||
await page.route('**/api/nodes/models', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(models) }))
|
||||
await page.goto('/app/nodes')
|
||||
await page.getByRole('tab', { name: 'Running models' }).click()
|
||||
await expect(page.getByRole('table', { name: 'Running models' }).locator('tbody tr')).toHaveCount(50)
|
||||
await expect(page.getByText('Page 1 of 20')).toBeVisible()
|
||||
await page.getByRole('button', { name: 'Next model page' }).click()
|
||||
await expect(page.getByText('Page 2 of 20')).toBeVisible()
|
||||
await page.getByRole('searchbox', { name: 'Search running models' }).fill('model-0000')
|
||||
await expect(page.locator('#fleet-models-panel').getByText('Page 1 of 1')).toBeVisible()
|
||||
await page.getByRole('searchbox', { name: 'Search running models' }).fill('')
|
||||
await page.getByRole('button', { name: /Sort by model/ }).click()
|
||||
await expect(page.getByRole('table', { name: 'Running models' }).locator('tbody tr').first()).toContainText('model-0999')
|
||||
})
|
||||
|
||||
})
|
||||
@@ -1,64 +1,35 @@
|
||||
import { test, expect } from './coverage-fixtures.js'
|
||||
|
||||
async function mockCluster(page, nodes) {
|
||||
await page.route('**/api/nodes', r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) }))
|
||||
await page.route('**/api/nodes/models', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.route('**/api/nodes/scheduling', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) }))
|
||||
}
|
||||
|
||||
test.describe('Nodes roster header', () => {
|
||||
test('shows a cluster pulse line and no stat-card grid', async ({ page }) => {
|
||||
test.describe('Nodes fleet roster', () => {
|
||||
test('uses the fleet response without prefetching models or backends', async ({ page }) => {
|
||||
const requests = []
|
||||
page.on('request', request => requests.push(request.url()))
|
||||
await mockCluster(page, [
|
||||
{ id: 'n1', name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy' },
|
||||
{ id: 'n2', name: 'beta', node_type: 'backend', address: '10.0.0.2:50051', status: 'draining' },
|
||||
{ id: 'n1', name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', model_count: 3 },
|
||||
{ id: 'a1', name: 'agent-1', node_type: 'agent', address: '10.0.0.9:50051', status: 'draining', model_count: 0 },
|
||||
])
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.locator('.cluster-pulse')).toBeVisible({ timeout: 15_000 })
|
||||
await expect(page.locator('.cluster-pulse')).toContainText('2 nodes')
|
||||
await expect(page.locator('.stat-grid')).toHaveCount(0)
|
||||
await expect(page.getByRole('table', { name: 'Fleet nodes' })).toBeVisible({ timeout: 15_000 })
|
||||
await expect(page.getByRole('tab', { name: 'Nodes' })).toHaveAttribute('aria-selected', 'true')
|
||||
await page.getByRole('tab', { name: 'Nodes' }).click()
|
||||
await expect(page.getByRole('row', { name: /alpha/ })).toContainText('3')
|
||||
expect(requests.some(url => url.includes('/api/nodes/models'))).toBe(false)
|
||||
expect(requests.some(url => /\/api\/nodes\/[^/]+\/backends/.test(url))).toBe(false)
|
||||
})
|
||||
|
||||
test('shows an approval callout for pending nodes', async ({ page }) => {
|
||||
await mockCluster(page, [{ id: 'n3', name: 'gamma', node_type: 'backend', address: '10.0.0.3:50051', status: 'pending' }])
|
||||
test('preserves the empty worker setup experience', async ({ page }) => {
|
||||
await mockCluster(page, [])
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.locator('.attention-callout')).toContainText('approval', { timeout: 15_000 })
|
||||
})
|
||||
})
|
||||
|
||||
test.describe('Nodes roster panels', () => {
|
||||
test('shows used and total system RAM reported by a worker', async ({ page }) => {
|
||||
await mockCluster(page, [
|
||||
{
|
||||
id: 'n1',
|
||||
name: 'alpha',
|
||||
node_type: 'backend',
|
||||
address: '10.0.0.1:50051',
|
||||
status: 'healthy',
|
||||
total_ram: 8_000_000_000,
|
||||
available_ram: 3_000_000_000,
|
||||
},
|
||||
])
|
||||
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.locator('.node-panel').filter({ hasText: 'alpha' })).toContainText('RAM 4.7 GB / 7.5 GB', { timeout: 15_000 })
|
||||
})
|
||||
|
||||
test('shows model chips without clicking and filters by type', async ({ page }) => {
|
||||
await page.route('**/api/nodes', r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([
|
||||
{ id: 'n1', name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy' },
|
||||
{ id: 'a1', name: 'agent-1', node_type: 'agent', address: '10.0.0.9:50051', status: 'healthy' },
|
||||
]) }))
|
||||
await page.route('**/api/nodes/models', r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([
|
||||
{ node_id: 'n1', model_name: 'llama-3.3', state: 'loaded', in_flight: 2, replica_index: 0 },
|
||||
]) }))
|
||||
await page.route('**/api/nodes/scheduling', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' }))
|
||||
|
||||
await page.goto('/app/nodes')
|
||||
// model chip visible without any expand click
|
||||
await expect(page.locator('.node-panel').filter({ hasText: 'alpha' }).getByText('llama-3.3')).toBeVisible({ timeout: 15_000 })
|
||||
// segmented filter: Agent shows the agent node, hides the backend node
|
||||
await page.getByRole('radio', { name: /Agent/ }).click()
|
||||
await expect(page.getByText('agent-1')).toBeVisible()
|
||||
await expect(page.getByText('alpha')).toHaveCount(0)
|
||||
await expect(page.getByText('No workers registered yet')).toBeVisible({ timeout: 15_000 })
|
||||
})
|
||||
|
||||
test('preserves the distributed-disabled setup experience', async ({ page }) => {
|
||||
await page.route('**/api/nodes', route => route.fulfill({ status: 503, body: 'Service Unavailable' }))
|
||||
await page.goto('/app/nodes')
|
||||
await expect(page.getByText('Distributed Mode Not Enabled')).toBeVisible({ timeout: 15_000 })
|
||||
})
|
||||
})
|
||||
@@ -1 +1 @@
|
||||
514
|
||||
512
|
||||
@@ -382,7 +382,6 @@
|
||||
padding-left: var(--spacing-md);
|
||||
padding-right: var(--spacing-md);
|
||||
}
|
||||
|
||||
.nav-external {
|
||||
font-size: 0.55rem;
|
||||
margin-left: auto;
|
||||
@@ -9062,8 +9061,8 @@ button.collapsible-header:focus-visible {
|
||||
}
|
||||
.console-rail-header__title { display: inline-flex; align-items: center; gap: var(--spacing-sm); }
|
||||
.console-rail-header__title i { color: var(--color-primary); font-size: 0.9rem; }
|
||||
.console-rail-toggle {
|
||||
display: none;
|
||||
.console-rail-toggle,
|
||||
.console-rail-collapse {
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
place-items: center;
|
||||
@@ -9073,7 +9072,10 @@ button.collapsible-header:focus-visible {
|
||||
color: var(--color-text-secondary);
|
||||
cursor: pointer;
|
||||
}
|
||||
.console-rail-toggle:hover { border-color: var(--color-border-strong); color: var(--color-text-primary); }
|
||||
.console-rail-toggle { display: none; }
|
||||
.console-rail-collapse { display: grid; }
|
||||
.console-rail-toggle:hover,
|
||||
.console-rail-collapse:hover { border-color: var(--color-border-strong); color: var(--color-text-primary); }
|
||||
.console-rail-groups { display: flex; flex-direction: column; gap: var(--spacing-xs); }
|
||||
.console-group { display: flex; flex-direction: column; gap: 1px; }
|
||||
.console-group + .console-group {
|
||||
@@ -9101,14 +9103,78 @@ button.collapsible-header:focus-visible {
|
||||
.console-rail .nav-item:hover:not(.active) { transform: translateX(2px); }
|
||||
.console-rail .nav-item.active { box-shadow: none; }
|
||||
.console-rail .nav-item.active .nav-icon { color: var(--color-primary); }
|
||||
.console-rail--collapsed {
|
||||
flex-basis: 60px;
|
||||
width: 60px;
|
||||
padding-inline: 6px;
|
||||
}
|
||||
.console-rail--collapsed .console-rail-header {
|
||||
justify-content: center;
|
||||
padding-inline: 0;
|
||||
}
|
||||
.console-rail--collapsed .console-rail-header__title,
|
||||
.console-rail--collapsed .console-group-title,
|
||||
.console-rail--collapsed .nav-label,
|
||||
.console-rail--collapsed .nav-external {
|
||||
position: absolute;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
padding: 0;
|
||||
margin: -1px;
|
||||
overflow: hidden;
|
||||
clip: rect(0, 0, 0, 0);
|
||||
white-space: nowrap;
|
||||
border: 0;
|
||||
}
|
||||
.console-rail--collapsed .console-group + .console-group {
|
||||
margin-top: var(--spacing-xs);
|
||||
padding-top: var(--spacing-xs);
|
||||
}
|
||||
.console-rail--collapsed .nav-item {
|
||||
justify-content: center;
|
||||
min-height: 38px;
|
||||
padding: 7px;
|
||||
}
|
||||
.console-rail--collapsed .nav-icon { width: auto; }
|
||||
.console-rail--collapsed .nav-signal {
|
||||
position: absolute;
|
||||
top: 1px;
|
||||
right: 1px;
|
||||
min-width: 14px;
|
||||
padding: 1px 3px;
|
||||
font-size: 0.5rem;
|
||||
line-height: 1.2;
|
||||
}
|
||||
.console-body {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
width: 100%;
|
||||
}
|
||||
@media (max-width: 768px) {
|
||||
.console-layout { flex-direction: column; padding: var(--spacing-sm); }
|
||||
.console-rail { position: static; flex-basis: auto; width: 100%; }
|
||||
.console-rail,
|
||||
.console-rail--collapsed { position: static; flex-basis: auto; width: 100%; padding: var(--spacing-sm); }
|
||||
.console-rail--collapsed .console-rail-header { justify-content: space-between; padding: var(--spacing-sm) var(--spacing-sm) var(--spacing-xs); }
|
||||
.console-rail--collapsed .console-rail-header__title,
|
||||
.console-rail--collapsed .console-group-title,
|
||||
.console-rail--collapsed .nav-label,
|
||||
.console-rail--collapsed .nav-external {
|
||||
position: static;
|
||||
width: auto;
|
||||
height: auto;
|
||||
padding: revert;
|
||||
margin: 0;
|
||||
overflow: visible;
|
||||
clip: auto;
|
||||
white-space: nowrap;
|
||||
border: 0;
|
||||
}
|
||||
.console-rail--collapsed .console-group-title { padding: var(--spacing-xs) var(--spacing-sm); }
|
||||
.console-rail--collapsed .nav-item { justify-content: flex-start; min-height: 44px; padding: 7px var(--spacing-sm); }
|
||||
.console-rail--collapsed .nav-icon { width: 18px; }
|
||||
.console-rail--collapsed .nav-signal { position: static; min-width: 0; padding: 1px 6px; font-size: var(--text-xs); line-height: inherit; }
|
||||
.console-rail-toggle { display: grid; }
|
||||
.console-rail-collapse { display: none; }
|
||||
.console-rail-groups { display: none; }
|
||||
/* Thirteen destinations stacked in one column is taller than a phone, so
|
||||
opening the menu used to push the page's own heading past the fold: the
|
||||
@@ -9677,6 +9743,202 @@ button.collapsible-header:focus-visible {
|
||||
.model-chip__state { opacity: 0.85; font-style: normal; }
|
||||
.node-filter { margin-bottom: var(--spacing-lg); }
|
||||
.node-detail__metrics { display: flex; gap: var(--spacing-xl); margin: var(--spacing-md) 0 var(--spacing-lg); flex-wrap: wrap; }
|
||||
.node-detail__metric-note { display: block; color: var(--color-text-muted); font-size: var(--text-xs); margin-top: 2px; }
|
||||
|
||||
/* Nodes fleet operations dashboard */
|
||||
.nodes-fleet-page { container-name: fleet-page; container-type: inline-size; isolation: isolate; position: relative; }
|
||||
.nodes-fleet-page--inspecting { container-type: normal; }
|
||||
.page-transition:has(.nodes-fleet-page--inspecting) { animation: none !important; transform: none !important; }
|
||||
.nodes-fleet-page__header { align-items: flex-start; margin-bottom: 18px; }
|
||||
.nodes-fleet-page__header .page-title { font-size: 1.75rem; letter-spacing: -.03em; }
|
||||
.nodes-fleet-page__header .page-header__supporting { font-size: var(--text-xs); margin-top: 3px; }
|
||||
.fleet-kicker { display: block; color: var(--color-text-muted); font-size: .625rem; font-weight: 650; letter-spacing: .08em; margin-bottom: 4px; text-transform: uppercase; }
|
||||
.fleet-overview {
|
||||
background: var(--color-bg-secondary);
|
||||
border: 1px solid var(--color-border-subtle);
|
||||
border-radius: var(--radius-lg);
|
||||
display: grid;
|
||||
grid-template-columns: minmax(265px, 1.45fr) repeat(4, minmax(145px, .8fr));
|
||||
margin-bottom: 12px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.fleet-overview__cell { height: 204px; min-width: 0; padding: 22px 20px; }
|
||||
.fleet-overview__cell + .fleet-overview__cell { border-left: 1px solid var(--color-border-subtle); }
|
||||
.fleet-health__headline { align-items: baseline; display: flex; gap: 8px; margin: 10px 0 22px; white-space: nowrap; }
|
||||
.fleet-health__headline strong { font-size: 1.625rem; letter-spacing: -.025em; }
|
||||
.fleet-health__headline span { color: var(--color-text-muted); font-size: .75rem; }
|
||||
.fleet-health__bar { display: block; height: 11px; margin-bottom: 17px; width: 100%; }
|
||||
.fleet-health__segment--healthy { fill: var(--color-success); }
|
||||
.fleet-health__segment--draining, .fleet-health__segment--pending { fill: var(--color-warning); }
|
||||
.fleet-health__segment--unhealthy { fill: var(--color-error); }
|
||||
.fleet-health__legend { display: grid; gap: 8px; grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||
.fleet-health__legend > div { min-width: 0; }
|
||||
.fleet-health__legend span { align-items: center; color: var(--color-text-muted); display: flex; font-size: .625rem; gap: 6px; white-space: nowrap; }
|
||||
.fleet-health__legend strong { display: block; font-size: .75rem; margin-top: 4px; white-space: nowrap; }
|
||||
.fleet-health__dot { background: var(--color-error); border-radius: 50%; display: inline-block; height: 7px; width: 7px; }
|
||||
.fleet-health__dot--healthy { background: var(--color-success); }
|
||||
.fleet-health__dot--draining { background: var(--color-warning); }
|
||||
.fleet-gauge { align-content: start; display: grid; justify-items: center; }
|
||||
.fleet-gauge .fleet-kicker { justify-self: start; white-space: nowrap; }
|
||||
.fleet-gauge__graphic { height: 73px; margin-top: 18px; position: relative; width: 136px; }
|
||||
.fleet-gauge__graphic svg { height: 73px; overflow: visible; width: 136px; }
|
||||
.fleet-gauge__graphic path { fill: none; stroke-linecap: butt; stroke-width: 7; }
|
||||
.fleet-gauge__track { stroke: var(--color-bg-tertiary); }
|
||||
.fleet-gauge__value { stroke: var(--fleet-gauge-color); }
|
||||
.fleet-gauge--vram { --fleet-gauge-color: var(--color-success); }
|
||||
.fleet-gauge--ram { --fleet-gauge-color: var(--color-primary); }
|
||||
.fleet-gauge--cpu { --fleet-gauge-color: var(--color-info); }
|
||||
.fleet-gauge--disk { --fleet-gauge-color: var(--color-info); }
|
||||
.fleet-gauge__graphic strong { bottom: 0; font-size: 1rem; left: 0; position: absolute; text-align: center; width: 136px; }
|
||||
.fleet-gauge__value-text { font-family: var(--font-mono); font-size: .6875rem; font-weight: 600; margin-top: 4px; text-align: center; white-space: nowrap; }
|
||||
.fleet-gauge__detail, .fleet-gauge__coverage { color: var(--color-text-muted); font-size: .5625rem; line-height: 1.35; margin-top: 3px; text-align: center; }
|
||||
.fleet-gauge__coverage { margin-top: 1px; }
|
||||
.fleet-attention { align-items: center; background: var(--color-warning-light); border: 1px solid var(--color-warning-border); border-radius: var(--radius-md); display: flex; gap: 18px; margin-bottom: 24px; min-height: 52px; padding: 8px 15px; }
|
||||
.fleet-attention__title { align-items: center; display: flex; flex: 0 0 auto; font-size: var(--text-xs); gap: 9px; }
|
||||
.fleet-attention__title i { color: var(--color-warning); }
|
||||
.fleet-attention__filters { align-items: center; display: flex; flex: 1; flex-wrap: wrap; gap: 6px; justify-content: flex-end; }
|
||||
.fleet-attention__filter { align-items: center; background: transparent; border: 1px solid transparent; border-radius: var(--radius-full); color: var(--color-text-secondary); cursor: pointer; display: flex; font: inherit; font-size: .625rem; gap: 6px; min-height: 28px; padding: 3px 9px; text-align: left; }
|
||||
.fleet-attention__filter:hover, .fleet-attention__filter.is-active { color: var(--color-primary); }
|
||||
.fleet-attention__filter:hover { background: var(--color-bg-hover); }
|
||||
.fleet-attention__filter.is-active { background: var(--color-bg-secondary); border-color: var(--color-border-strong); box-shadow: 0 1px 2px rgba(0, 0, 0, .12); }
|
||||
.fleet-attention__filter strong { color: var(--color-text-primary); }
|
||||
.fleet-workbench { background: var(--color-bg-secondary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-lg); container-name: fleet-workbench; container-type: inline-size; overflow: hidden; }
|
||||
.fleet-workbench__tabs { align-items: stretch; background: var(--color-bg-secondary); border-bottom: 1px solid var(--color-border-subtle); display: flex; min-height: 43px; padding: 0 11px; }
|
||||
.fleet-workbench__tabs button { align-items: center; background: transparent; border: 0; border-bottom: 2px solid transparent; color: var(--color-text-muted); cursor: pointer; display: flex; font: inherit; font-size: var(--text-xs); font-weight: 600; gap: 7px; margin-bottom: -1px; padding: 0 11px; }
|
||||
.fleet-workbench__tabs button:hover { color: var(--color-text-primary); }
|
||||
.fleet-workbench__tabs button:focus-visible { border-radius: var(--radius-sm); box-shadow: inset 0 0 0 2px var(--color-primary); outline: none; }
|
||||
.fleet-workbench__tabs button.is-active { border-bottom-color: var(--color-primary); color: var(--color-text-primary); }
|
||||
.fleet-workbench__tabs button span { background: var(--color-bg-tertiary); border-radius: var(--radius-full); color: var(--color-text-muted); font-family: var(--font-mono); font-size: .5625rem; min-width: 19px; padding: 2px 6px; text-align: center; }
|
||||
.fleet-workbench__layout { min-width: 0; }
|
||||
.fleet-workbench__fleet { min-width: 0; }
|
||||
.fleet-toolbar { border-bottom: 1px solid var(--color-border-subtle); display: grid; grid-template-columns: minmax(190px, 1fr) repeat(3, minmax(112px, auto)); gap: 7px; padding: 10px 11px; }
|
||||
.fleet-toolbar__search { height: 34px; min-width: 0; }
|
||||
.fleet-select-wrap { display: block; min-width: 0; position: relative; }
|
||||
.fleet-select { appearance: none; background: var(--color-bg-primary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-md); color: var(--color-text-secondary); cursor: pointer; font: inherit; font-size: var(--text-xs); height: 34px; padding: 0 28px 0 10px; width: 100%; }
|
||||
.fleet-select:hover { border-color: var(--color-border-strong); }
|
||||
.fleet-select:focus-visible { border-color: var(--color-primary); box-shadow: 0 0 0 3px var(--color-primary-light); outline: none; }
|
||||
.fleet-select__chevron { color: var(--color-text-muted); font-size: .625rem; pointer-events: none; position: absolute; right: 10px; top: 12px; }
|
||||
.fleet-bulkbar { align-items: center; background: var(--color-primary-light); border-bottom: 1px solid var(--color-border-strong); display: flex; flex-wrap: wrap; gap: var(--spacing-xs); min-height: 42px; padding: 5px 11px; }
|
||||
.fleet-bulkbar strong { font-size: var(--text-xs); margin-right: var(--spacing-xs); }
|
||||
.fleet-bulkbar__clear { background: none; border: 0; color: var(--color-primary); cursor: pointer; font: inherit; font-size: var(--text-xs); }
|
||||
.fleet-bulkbar__count { color: var(--color-text-muted); font-size: var(--text-xs); margin-left: auto; }
|
||||
.fleet-table-wrap { overflow-x: auto; }
|
||||
.fleet-table { border-collapse: collapse; font-size: var(--text-xs); min-width: 820px; width: 100%; }
|
||||
.fleet-table th, .fleet-table td { border-bottom: 1px solid var(--color-border-subtle); padding: 10px 12px; text-align: left; vertical-align: middle; }
|
||||
.fleet-table thead th { background: var(--color-bg-secondary); color: var(--color-text-muted); font-weight: 600; }
|
||||
.fleet-table__check { width: 34px; }
|
||||
.fleet-table__sort { background: none; border: 0; color: inherit; cursor: pointer; font: inherit; font-weight: inherit; padding: 0; }
|
||||
.fleet-table__row { cursor: pointer; height: 67px; transition: background var(--duration-fast) var(--ease-out, ease-out); }
|
||||
.fleet-table__row:hover, .fleet-table__row:focus { background: var(--color-bg-hover); outline: none; }
|
||||
.fleet-table__row.is-selected { background: var(--color-primary-light); box-shadow: inset 2px 0 var(--color-primary); }
|
||||
.fleet-table__node { background: none; border: 0; color: var(--color-text-primary); cursor: pointer; display: block; font: inherit; font-weight: 600; padding: 0; text-align: left; }
|
||||
.fleet-table__node + span, .fleet-table__subvalue { color: var(--color-text-muted); display: block; font-size: .625rem; margin-top: 1px; }
|
||||
.fleet-table__approve { background: none; border: 0; color: var(--color-primary); cursor: pointer; display: block; font: inherit; font-size: .625rem; font-weight: 650; margin-top: 5px; padding: 0; }
|
||||
.fleet-table__unknown { color: var(--color-text-muted); }
|
||||
.fleet-table__resource { align-items: center; display: grid; gap: 5px; grid-template-columns: minmax(45px, 1fr) auto; min-width: 92px; }
|
||||
.fleet-table__resource > span:last-child { color: var(--color-text-muted); font-family: var(--font-mono); font-size: .5625rem; white-space: nowrap; }
|
||||
.fleet-table__resource-track { appearance: none; background: var(--color-bg-tertiary); border: 0; border-radius: var(--radius-full); display: block; height: 5px; overflow: hidden; width: 100%; }
|
||||
.fleet-table__resource-track::-webkit-progress-bar { background: var(--color-bg-tertiary); }
|
||||
.fleet-table__resource-track::-webkit-progress-value { background: var(--fleet-resource-color); border-radius: var(--radius-full); }
|
||||
.fleet-table__resource-track::-moz-progress-bar { background: var(--fleet-resource-color); border-radius: var(--radius-full); }
|
||||
.fleet-table__resource--vram { --fleet-resource-color: var(--color-success); }
|
||||
.fleet-table__resource--ram { --fleet-resource-color: var(--color-primary); }
|
||||
.fleet-table__capacity { display: grid; gap: 7px; min-width: 205px; }
|
||||
.fleet-table__capacity-row { align-items: center; display: grid; gap: 8px; grid-template-columns: 38px minmax(0, 1fr); }
|
||||
.fleet-table__capacity-row > b { color: var(--color-text-muted); font-size: .5625rem; font-weight: 500; }
|
||||
.fleet-table__group th { background: var(--color-bg-tertiary); color: var(--color-text-primary); padding: 7px 10px; }
|
||||
.fleet-table__group th, .fleet-table__group label { align-items: center; display: flex; gap: var(--spacing-xs); }
|
||||
.fleet-table__group span { color: var(--color-text-muted); font-weight: 400; margin-left: auto; }
|
||||
.fleet-table__empty { color: var(--color-text-muted); padding: var(--spacing-xl); text-align: center; }
|
||||
.fleet-pagination { align-items: center; display: flex; gap: var(--spacing-xs); justify-content: flex-end; min-height: 45px; padding: 5px 11px; }
|
||||
.fleet-pagination span { color: var(--color-text-muted); font-size: var(--text-xs); margin-right: var(--spacing-xs); }
|
||||
.model-workbench__scope { align-items: center; background: var(--color-bg-tertiary); border-bottom: 1px solid var(--color-border-subtle); display: flex; justify-content: space-between; min-height: 48px; padding: 7px 12px; }
|
||||
.model-workbench__scope > div { display: grid; gap: 1px; }
|
||||
.model-workbench__scope strong { color: var(--color-text-primary); font-size: var(--text-sm); }
|
||||
.model-workbench__scope span { color: var(--color-text-muted); font-size: .625rem; }
|
||||
.model-toolbar { border-bottom: 1px solid var(--color-border-subtle); padding: 9px 11px; }
|
||||
.model-toolbar .fleet-toolbar__search { max-width: 430px; width: 100%; }
|
||||
.model-workbench__state { align-items: center; color: var(--color-text-muted); display: flex; flex-direction: column; gap: 6px; justify-content: center; min-height: 300px; padding: var(--spacing-xl); text-align: center; }
|
||||
.model-workbench__state > i { color: var(--color-text-muted); font-size: var(--text-xl); }
|
||||
.model-workbench__state strong { color: var(--color-text-primary); font-size: var(--text-base); }
|
||||
.model-workbench__state span { font-size: var(--text-xs); max-width: 420px; }
|
||||
.model-workbench__state .btn { margin-top: 5px; }
|
||||
.model-workbench__state--error > i { color: var(--color-error); }
|
||||
.model-fleet-table { min-width: 720px; }
|
||||
.model-fleet-table th:first-child, .model-fleet-table td:first-child { width: 37%; }
|
||||
.model-fleet-table .model-fleet-table__actions { padding-left: 4px; padding-right: 8px; text-align: right; width: 38px; }
|
||||
.model-fleet-table .fleet-table__row:hover .action-menu__trigger,
|
||||
.model-fleet-table .action-menu__trigger:focus-visible,
|
||||
.model-fleet-table .action-menu__trigger.is-open { opacity: 1; }
|
||||
.model-backend-list { display: flex; flex-wrap: wrap; gap: 4px; }
|
||||
.model-backend-list > span:not(.fleet-table__unknown, .text-muted) { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-full); color: var(--color-text-secondary); font-family: var(--font-mono); font-size: .5625rem; padding: 2px 6px; }
|
||||
.node-inspector__scrim { display: none; }
|
||||
.node-inspector { animation: node-inspector-in 180ms var(--ease-out, ease-out) both; background: var(--color-bg-primary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-xl) 0 0 var(--radius-xl); bottom: var(--spacing-md); box-shadow: -18px 0 42px rgba(0, 0, 0, .24); display: grid; grid-template-rows: auto minmax(0, 1fr) auto; overflow: hidden; position: fixed; right: var(--spacing-md); top: var(--spacing-md); width: 360px; z-index: 80; }
|
||||
@keyframes node-inspector-in { from { opacity: 0; transform: translateX(18px); } to { opacity: 1; transform: translateX(0); } }
|
||||
.node-inspector__header { background: linear-gradient(180deg, var(--color-bg-secondary), var(--color-bg-primary)); border-bottom: 1px solid var(--color-border-subtle); padding: 18px 22px 17px; }
|
||||
.node-inspector__topbar { align-items: center; display: flex; justify-content: space-between; min-height: 30px; }
|
||||
.node-inspector__topbar .fleet-kicker { margin: 0; }
|
||||
.node-inspector__header h2 { font-size: 1.25rem; letter-spacing: -.025em; line-height: 1.2; margin: 9px 0 0; overflow-wrap: anywhere; }
|
||||
.node-inspector__header > p, .node-inspector__identity > p { color: var(--color-text-muted); font-size: .6875rem; margin: 4px 0 9px; }
|
||||
.node-inspector__identity .status-pill { margin: 0; }
|
||||
.node-inspector__body { min-height: 0; overflow-y: auto; overscroll-behavior: contain; padding: 0 22px; scrollbar-gutter: stable; }
|
||||
.node-inspector__section { border-bottom: 1px solid var(--color-border-subtle); padding: 15px 0; }
|
||||
.node-inspector__section:last-of-type { border-bottom: 0; }
|
||||
.node-inspector__section h3 { color: var(--color-text-muted); font-size: .625rem; font-weight: 650; letter-spacing: .08em; margin: 0 0 9px; text-transform: uppercase; }
|
||||
.node-inspector__address { color: var(--color-text-secondary); font-family: var(--font-mono); font-size: .625rem; overflow-wrap: anywhere; }
|
||||
.node-inspector__metrics { display: grid; gap: 0; margin: 0; }
|
||||
.node-inspector__metrics > div { display: grid; font-size: var(--text-xs); grid-template-columns: 92px 1fr; padding: 5px 0; }
|
||||
.node-inspector__metrics dt { color: var(--color-text-muted); }
|
||||
.node-inspector__metrics dd { margin: 0; text-align: right; }
|
||||
.node-inspector__labels { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 8px; }
|
||||
.node-inspector__labels > span:not(.text-muted) { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-full); color: var(--color-text-secondary); font-family: var(--font-mono); font-size: .5625rem; padding: 2px 7px; }
|
||||
.node-inspector__resource { margin: 10px 0 13px; }
|
||||
.node-inspector__resource-label { align-items: baseline; display: flex; font-size: var(--text-xs); justify-content: space-between; margin-bottom: 6px; }
|
||||
.node-inspector__resource-label strong span, .node-inspector__resource-label > span { color: var(--color-text-muted); font-size: .625rem; font-weight: 400; }
|
||||
.node-inspector__resource-track { appearance: none; background: var(--color-bg-tertiary); border: 0; border-radius: var(--radius-full); display: block; height: 7px; overflow: hidden; width: 100%; }
|
||||
.node-inspector__resource-track::-webkit-progress-bar { background: var(--color-bg-tertiary); }
|
||||
.node-inspector__resource-track::-webkit-progress-value { background: var(--fleet-resource-color); border-radius: var(--radius-full); }
|
||||
.node-inspector__resource-track::-moz-progress-bar { background: var(--fleet-resource-color); border-radius: var(--radius-full); }
|
||||
.node-inspector__resource--vram { --fleet-resource-color: var(--color-success); }
|
||||
.node-inspector__resource--ram { --fleet-resource-color: var(--color-primary); }
|
||||
.node-inspector__actions { background: var(--color-bg-secondary); border-top: 1px solid var(--color-border-subtle); box-shadow: 0 -8px 18px rgba(0, 0, 0, .06); display: grid; gap: var(--spacing-xs); grid-template-columns: 1fr 1fr; padding: 14px 22px; }
|
||||
.node-inspector__actions--single { grid-template-columns: 1fr; }
|
||||
.node-inspector__actions .btn { justify-content: center; min-width: 0; }
|
||||
.node-inspector__back { align-items: center; background: transparent; border: 0; color: var(--color-primary); cursor: pointer; display: flex; font: inherit; font-size: var(--text-xs); gap: 6px; max-width: 285px; overflow: hidden; padding: 3px 0; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.node-inspector__back:focus-visible { border-radius: var(--radius-sm); outline: 2px solid var(--color-primary); outline-offset: 3px; }
|
||||
.model-inspector__backends { margin-top: 10px; }
|
||||
.model-inspector__nodes { display: grid; gap: 9px; }
|
||||
.model-inspector__node { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-md); padding: 10px; }
|
||||
.model-inspector__node-heading { align-items: center; display: flex; gap: 7px; justify-content: space-between; }
|
||||
.model-inspector__node-heading button { background: transparent; border: 0; color: var(--color-primary); cursor: pointer; font: inherit; font-size: var(--text-sm); font-weight: 650; overflow: hidden; padding: 0; text-align: left; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.model-inspector__node-heading strong { font-size: var(--text-sm); overflow-wrap: anywhere; }
|
||||
.model-inspector__node > p { color: var(--color-text-muted); font-size: .625rem; margin: 5px 0 8px; }
|
||||
.model-inspector__node ul { border-top: 1px solid var(--color-border-subtle); list-style: none; margin: 0; padding: 5px 0 0; }
|
||||
.model-inspector__node li { align-items: baseline; display: grid; font-size: .625rem; gap: 7px; grid-template-columns: auto minmax(0, 1fr); padding-top: 4px; }
|
||||
.model-inspector__node li span { color: var(--color-text-muted); }
|
||||
.model-inspector__node code { color: var(--color-text-secondary); font-size: .5625rem; overflow: hidden; text-align: right; text-overflow: ellipsis; white-space: nowrap; }
|
||||
|
||||
@container fleet-page (max-width: 760px) {
|
||||
.fleet-overview { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.fleet-overview__cell { min-height: 146px; }
|
||||
.fleet-overview__cell + .fleet-overview__cell { border-left: 0; }
|
||||
.fleet-overview__cell:nth-child(even) { border-left: 1px solid var(--color-border-subtle); }
|
||||
.fleet-overview__cell:nth-child(n + 3) { border-top: 1px solid var(--color-border-subtle); }
|
||||
.fleet-health { grid-column: 1 / -1; }
|
||||
.fleet-health { min-height: 126px; }
|
||||
.fleet-attention { align-items: flex-start; flex-direction: column; gap: 7px; }
|
||||
.fleet-attention__filters { justify-content: flex-start; }
|
||||
.fleet-toolbar { grid-template-columns: 1fr 1fr; }
|
||||
.fleet-toolbar__search { grid-column: 1 / -1; }
|
||||
}
|
||||
@media (max-width: 768px) {
|
||||
.node-inspector__scrim { animation: node-inspector-scrim-in 180ms ease-out both; background: rgba(7, 10, 18, .52); display: block; inset: 0; position: fixed; z-index: 79; }
|
||||
.node-inspector { border-bottom: 0; border-radius: 0; border-top: 0; bottom: 0; max-width: 100vw; position: fixed; top: 0; width: min(390px, 100vw); z-index: 80; }
|
||||
.node-inspector__header { padding-left: 18px; padding-right: 18px; padding-top: max(16px, env(safe-area-inset-top)); }
|
||||
.node-inspector__body { padding-left: 18px; padding-right: 18px; }
|
||||
.node-inspector__actions { padding-bottom: max(14px, env(safe-area-inset-bottom)); padding-left: 18px; padding-right: 18px; }
|
||||
}
|
||||
@keyframes node-inspector-scrim-in { from { opacity: 0; } to { opacity: 1; } }
|
||||
|
||||
/* Rendered Markdown ---------------------------------------------------------
|
||||
Gallery descriptions, backend descriptions and voice notes are all
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useRef, useState, useEffect, useCallback } from 'react'
|
||||
import { useRef, useState, useEffect, useCallback, useId } from 'react'
|
||||
import Popover from './Popover'
|
||||
|
||||
// ActionMenu renders a kebab (three-dot) button that opens a popover with a
|
||||
@@ -20,6 +20,7 @@ import Popover from './Popover'
|
||||
// Escape — close, return focus to trigger
|
||||
export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel, compact = false }) {
|
||||
const triggerRef = useRef(null)
|
||||
const menuId = useId()
|
||||
const [open, setOpen] = useState(false)
|
||||
const [activeIdx, setActiveIdx] = useState(-1)
|
||||
|
||||
@@ -48,7 +49,12 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel,
|
||||
}
|
||||
|
||||
const handleMenuKeyDown = (e) => {
|
||||
if (e.key === 'ArrowDown') {
|
||||
if (e.key === 'Escape') {
|
||||
// Keep the same Escape press from also closing a surrounding inspector.
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
close()
|
||||
} else if (e.key === 'ArrowDown') {
|
||||
e.preventDefault()
|
||||
setActiveIdx(i => Math.min(interactive.length - 1, (i < 0 ? -1 : i) + 1))
|
||||
} else if (e.key === 'ArrowUp') {
|
||||
@@ -65,7 +71,7 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel,
|
||||
const item = interactive[activeIdx]
|
||||
if (item && !item.disabled) {
|
||||
close()
|
||||
item.onClick?.()
|
||||
item.onClick?.(triggerRef.current)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -92,6 +98,7 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel,
|
||||
<div
|
||||
role="menu"
|
||||
aria-label={ariaLabel}
|
||||
aria-activedescendant={activeIdx >= 0 ? `${menuId}-item-${activeIdx}` : undefined}
|
||||
className="action-menu"
|
||||
onKeyDown={handleMenuKeyDown}
|
||||
// Capture focus when the menu opens so arrow keys work without the
|
||||
@@ -118,8 +125,10 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel,
|
||||
return (
|
||||
<button
|
||||
key={item.key}
|
||||
id={`${menuId}-item-${idx}`}
|
||||
type="button"
|
||||
role="menuitem"
|
||||
tabIndex={-1}
|
||||
disabled={item.disabled}
|
||||
className={`action-menu__item${item.danger ? ' is-danger' : ''}${active ? ' is-active' : ''}`}
|
||||
onMouseEnter={() => setActiveIdx(idx)}
|
||||
@@ -127,7 +136,7 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel,
|
||||
e.stopPropagation()
|
||||
if (item.disabled) return
|
||||
close()
|
||||
item.onClick?.()
|
||||
item.onClick?.(triggerRef.current)
|
||||
}}
|
||||
>
|
||||
{item.icon && <i className={`fas ${item.icon} action-menu__icon`} aria-hidden="true" />}
|
||||
|
||||
@@ -6,8 +6,10 @@ export default function ConfirmDialog({
|
||||
title,
|
||||
message,
|
||||
confirmLabel,
|
||||
pendingLabel,
|
||||
cancelLabel,
|
||||
danger = false,
|
||||
pending = false,
|
||||
onConfirm,
|
||||
onCancel,
|
||||
}) {
|
||||
@@ -30,7 +32,7 @@ export default function ConfirmDialog({
|
||||
const getFocusable = () => dialog.querySelectorAll(focusableSelector)
|
||||
|
||||
const handleKeyDown = (e) => {
|
||||
if (e.key === 'Escape') {
|
||||
if (e.key === 'Escape' && !pending) {
|
||||
onCancel?.()
|
||||
return
|
||||
}
|
||||
@@ -54,7 +56,7 @@ export default function ConfirmDialog({
|
||||
|
||||
document.addEventListener('keydown', handleKeyDown)
|
||||
return () => document.removeEventListener('keydown', handleKeyDown)
|
||||
}, [open, onCancel])
|
||||
}, [open, onCancel, pending])
|
||||
|
||||
if (!open) return null
|
||||
|
||||
@@ -62,7 +64,7 @@ export default function ConfirmDialog({
|
||||
const bodyId = 'confirm-dialog-body'
|
||||
|
||||
return (
|
||||
<div className="confirm-dialog-backdrop" onClick={onCancel}>
|
||||
<div className="confirm-dialog-backdrop" onClick={pending ? undefined : onCancel}>
|
||||
<div
|
||||
ref={dialogRef}
|
||||
className="confirm-dialog"
|
||||
@@ -78,15 +80,16 @@ export default function ConfirmDialog({
|
||||
</div>
|
||||
{message && <div id={bodyId} className="confirm-dialog-body">{message}</div>}
|
||||
<div className="confirm-dialog-actions">
|
||||
<button className="btn btn-secondary btn-sm" onClick={onCancel}>
|
||||
<button className="btn btn-secondary btn-sm" disabled={pending} onClick={onCancel}>
|
||||
{cancelText}
|
||||
</button>
|
||||
<button
|
||||
ref={confirmRef}
|
||||
className={`btn btn-sm ${danger ? 'btn-danger' : 'btn-primary'}`}
|
||||
disabled={pending}
|
||||
onClick={onConfirm}
|
||||
>
|
||||
{confirmText}
|
||||
{pending ? (pendingLabel ?? confirmText) : confirmText}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -20,6 +20,7 @@ import { createPortal } from 'react-dom'
|
||||
// ariaLabel: accessible label for the dialog
|
||||
export default function Popover({ anchor, open, onClose, children, ariaLabel }) {
|
||||
const popoverRef = useRef(null)
|
||||
const wasOpenRef = useRef(open)
|
||||
const [pos, setPos] = useState({ top: 0, left: 0, flipped: false })
|
||||
|
||||
// Compute position from the anchor's bounding box whenever we open or the
|
||||
@@ -68,16 +69,21 @@ export default function Popover({ anchor, open, onClose, children, ariaLabel })
|
||||
}
|
||||
}, [open, onClose, anchor])
|
||||
|
||||
// Return focus to the trigger when the popover closes — keyboard users
|
||||
// shouldn't have to tab back through the whole page to find their spot.
|
||||
// Return focus only after an open popover closes. Running this on initial
|
||||
// mount makes every closed row menu contend for focus, and the final row
|
||||
// wins. If the close also opened a modal, let that modal keep focus.
|
||||
useEffect(() => {
|
||||
if (!open && anchor?.current) {
|
||||
// requestAnimationFrame so the close is painted before focus jumps;
|
||||
// otherwise screen readers announce the trigger mid-transition.
|
||||
// preventScroll: focusing the trigger must not yank the page scroll.
|
||||
const raf = requestAnimationFrame(() => anchor.current?.focus?.({ preventScroll: true }))
|
||||
return () => cancelAnimationFrame(raf)
|
||||
}
|
||||
const wasOpen = wasOpenRef.current
|
||||
wasOpenRef.current = open
|
||||
if (open || !wasOpen || !anchor?.current) return undefined
|
||||
|
||||
const raf = requestAnimationFrame(() => {
|
||||
const modals = document.querySelectorAll('[aria-modal="true"]')
|
||||
const topmostModal = modals[modals.length - 1]
|
||||
if (topmostModal?.contains(document.activeElement)) return
|
||||
anchor.current?.focus?.({ preventScroll: true })
|
||||
})
|
||||
return () => cancelAnimationFrame(raf)
|
||||
}, [open, anchor])
|
||||
|
||||
if (!open) return null
|
||||
|
||||
@@ -18,13 +18,14 @@ let lastConsoleId = null
|
||||
// the correct (gated) item set immediately instead of flashing the wrong set
|
||||
// while a fresh fetch resolves on every sub-navigation.
|
||||
let featuresCache = {}
|
||||
const CONSOLE_RAIL_COLLAPSED_KEY = 'localai_console_rail_collapsed'
|
||||
|
||||
// Generic secondary-rail layout shared by the Build and Operate consoles.
|
||||
// Driven entirely by a config from consoleConfig.js, so the rail, its gating,
|
||||
// and the sidebar entry that opens it stay in sync. Mounted as a PATHLESS
|
||||
// route in router.jsx — wrapped pages keep their existing flat URLs.
|
||||
|
||||
function RailItem({ item, label }) {
|
||||
function RailItem({ item, label, collapsed }) {
|
||||
// Null outside Operate, where no provider is mounted — the rail then renders
|
||||
// exactly as it did before signals existed.
|
||||
const summary = useOperateSummary()
|
||||
@@ -32,7 +33,7 @@ function RailItem({ item, label }) {
|
||||
|
||||
if (item.external) {
|
||||
return (
|
||||
<a className="nav-item" href={apiUrl(item.href)} target="_blank" rel="noopener noreferrer">
|
||||
<a className="nav-item" href={apiUrl(item.href)} target="_blank" rel="noopener noreferrer" aria-label={collapsed ? label : undefined} title={collapsed ? label : undefined}>
|
||||
<i className={`${item.icon} nav-icon`} />
|
||||
<span className="nav-label">{label}</span>
|
||||
<i className="fas fa-external-link-alt nav-external" />
|
||||
@@ -45,6 +46,8 @@ function RailItem({ item, label }) {
|
||||
className={({ isActive }) => `nav-item ${isActive ? 'active' : ''}`}
|
||||
onMouseEnter={() => preloadRoute(item.path)}
|
||||
onFocus={() => preloadRoute(item.path)}
|
||||
aria-label={collapsed ? label : undefined}
|
||||
title={collapsed ? label : undefined}
|
||||
>
|
||||
<i className={`${item.icon} nav-icon`} />
|
||||
<span className="nav-label">{label}</span>
|
||||
@@ -61,6 +64,9 @@ function ConsoleLayoutInner({ config }) {
|
||||
const { isAdmin, authEnabled, hasFeature } = useAuth()
|
||||
const [features, setFeatures] = useState(featuresCache)
|
||||
const [railOpen, setRailOpen] = useState(false)
|
||||
const [railCollapsed, setRailCollapsed] = useState(() => {
|
||||
try { return localStorage.getItem(CONSOLE_RAIL_COLLAPSED_KEY) === 'true' } catch (_) { return false }
|
||||
})
|
||||
const location = useLocation()
|
||||
// Forward the App-level outlet context (e.g. addToast) — a nested bare
|
||||
// <Outlet/> would otherwise shadow it with undefined and crash pages.
|
||||
@@ -81,9 +87,15 @@ function ConsoleLayoutInner({ config }) {
|
||||
|
||||
const auth = { isAdmin, authEnabled, hasFeature, features }
|
||||
|
||||
const toggleRailCollapsed = () => {
|
||||
const next = !railCollapsed
|
||||
try { localStorage.setItem(CONSOLE_RAIL_COLLAPSED_KEY, String(next)) } catch (_) { /* ignore */ }
|
||||
setRailCollapsed(next)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="console-layout">
|
||||
<nav className={`console-rail${entering ? ' console-rail--enter' : ''}${railOpen ? ' console-rail--open' : ''}`} aria-label={t(config.titleKey)}>
|
||||
<nav className={`console-rail${entering ? ' console-rail--enter' : ''}${railOpen ? ' console-rail--open' : ''}${railCollapsed ? ' console-rail--collapsed' : ''}`} aria-label={t(config.titleKey)}>
|
||||
<div className="console-rail-header">
|
||||
<span className="console-rail-header__title">
|
||||
<i className={config.icon} aria-hidden="true" />
|
||||
@@ -99,6 +111,16 @@ function ConsoleLayoutInner({ config }) {
|
||||
>
|
||||
<i className={`fas fa-chevron-${railOpen ? 'up' : 'down'}`} aria-hidden="true" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="console-rail-collapse"
|
||||
aria-pressed={railCollapsed}
|
||||
aria-label={t(railCollapsed ? 'console.expandNavigation' : 'console.collapseNavigation', { section: t(config.titleKey) })}
|
||||
title={t(railCollapsed ? 'console.expandNavigation' : 'console.collapseNavigation', { section: t(config.titleKey) })}
|
||||
onClick={toggleRailCollapsed}
|
||||
>
|
||||
<i className={`fas fa-chevron-${railCollapsed ? 'right' : 'left'}`} aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
<div id={`console-rail-groups-${config.id}`} className="console-rail-groups">
|
||||
{config.groups.map((group, gi) => {
|
||||
@@ -108,7 +130,7 @@ function ConsoleLayoutInner({ config }) {
|
||||
<div key={group.titleKey || gi} className="console-group">
|
||||
{group.titleKey && <div className="console-group-title">{t(group.titleKey)}</div>}
|
||||
{items.map(item => (
|
||||
<RailItem key={item.path || item.href} item={item} label={t(item.labelKey)} />
|
||||
<RailItem key={item.path || item.href} item={item} label={t(item.labelKey)} collapsed={railCollapsed} />
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import { formatCapacity } from './nodeStatus'
|
||||
|
||||
const ATTENTION = [
|
||||
['all', 'Needs attention'],
|
||||
['pending', 'Pending approval'],
|
||||
['offlineOrUnhealthy', 'Offline / unhealthy'],
|
||||
['lowVRAM', 'Low VRAM'],
|
||||
['lowRAM', 'Low RAM'],
|
||||
['lowDisk', 'Low models disk'],
|
||||
]
|
||||
|
||||
function CapacityGauge({ label, metric, cpu = false, tone }) {
|
||||
const reporting = metric.reportingCount > 0
|
||||
const percent = reporting ? Math.round(metric.usagePercent) : 0
|
||||
const value = cpu
|
||||
? `${Number(metric.busyCoreEquivalents.toFixed(1))} busy / ${metric.totalLogicalCores} cores`
|
||||
: formatCapacity(metric.used, metric.total)
|
||||
const available = cpu
|
||||
? `${Number(metric.idleCoreEquivalents.toFixed(1))} idle · load ${metric.load1.toFixed(2)}`
|
||||
: reporting ? `${formatCapacity(metric.available, metric.total).split(' / ')[0]} available` : 'No data'
|
||||
|
||||
return (
|
||||
<article className={`fleet-gauge fleet-gauge--${tone} fleet-overview__cell`} aria-label={`${label} capacity`}>
|
||||
<div className="fleet-kicker">{label} capacity</div>
|
||||
<div className="fleet-gauge__graphic" aria-hidden="true">
|
||||
<svg viewBox="0 0 100 54">
|
||||
<path className="fleet-gauge__track" d="M7 50 A43 43 0 0 1 93 50" pathLength="100" />
|
||||
{reporting && <path className="fleet-gauge__value" d="M7 50 A43 43 0 0 1 93 50" pathLength="100" strokeDasharray={`${percent} 100`} />}
|
||||
</svg>
|
||||
<strong>{reporting ? `${percent}%` : '—'}</strong>
|
||||
</div>
|
||||
<div className="fleet-gauge__value-text">{reporting ? value : 'No data'}</div>
|
||||
<div className="fleet-gauge__detail">{available}</div>
|
||||
<span className="sr-only">Capacity coverage: {metric.reportingCount} of {metric.reportingCount + metric.unknownCount} nodes reporting; {metric.unknownCount} unknown.</span>
|
||||
{metric.unknownCount > 0 && <div className="fleet-gauge__coverage">{metric.unknownCount} node{metric.unknownCount === 1 ? '' : 's'} unavailable</div>}
|
||||
</article>
|
||||
)
|
||||
}
|
||||
|
||||
export default function ClusterOverview({ summary, activeAttention, onAttentionSelect }) {
|
||||
const { health } = summary
|
||||
const total = Math.max(health.total, 1)
|
||||
const segments = [
|
||||
['healthy', health.healthy],
|
||||
['draining', health.draining],
|
||||
['unhealthy', health.pending + health.unhealthy + health.offline + health.other],
|
||||
]
|
||||
let cursor = 0
|
||||
|
||||
return <>
|
||||
<section className="fleet-overview" aria-label="Fleet overview">
|
||||
<div className="fleet-health fleet-overview__cell" aria-label="Fleet health summary" aria-live="polite">
|
||||
<span className="fleet-kicker">Fleet health</span>
|
||||
<div className="fleet-health__headline"><strong>{health.healthy} healthy</strong><span>of {health.total} nodes</span></div>
|
||||
<svg className="fleet-health__bar" viewBox="0 0 100 4" preserveAspectRatio="none" aria-hidden="true">
|
||||
{segments.map(([status, count]) => {
|
||||
const start = cursor
|
||||
const width = count / total * 100
|
||||
cursor += width
|
||||
return <rect key={status} className={`fleet-health__segment fleet-health__segment--${status}`} x={start} y="0" width={width} height="4" />
|
||||
})}
|
||||
</svg>
|
||||
<div className="fleet-health__legend">
|
||||
{segments.map(([status, count]) => {
|
||||
const label = status === 'unhealthy' ? 'Attention' : status[0].toUpperCase() + status.slice(1)
|
||||
const percentage = health.total ? (count / health.total * 100).toFixed(1) : '0.0'
|
||||
return <div key={status}><span><i className={`fleet-health__dot fleet-health__dot--${status}`} />{label}</span><strong>{count} · {percentage}%</strong></div>
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<CapacityGauge label="VRAM" metric={summary.vram} tone="vram" />
|
||||
<CapacityGauge label="RAM" metric={summary.ram} tone="ram" />
|
||||
<CapacityGauge label="CPU" metric={summary.cpu} cpu tone="cpu" />
|
||||
<CapacityGauge label="Models disk" metric={summary.disk} tone="disk" />
|
||||
|
||||
</section>
|
||||
<aside className="fleet-attention" aria-label="Attention queue">
|
||||
<span className="fleet-attention__title"><i className="fas fa-triangle-exclamation" aria-hidden="true" /><strong>{summary.attentionNodeCount} node{summary.attentionNodeCount === 1 ? '' : 's'} need attention</strong></span>
|
||||
<div className="fleet-attention__filters">
|
||||
{ATTENTION.map(([key, label]) => {
|
||||
const count = key === 'all' ? summary.attentionNodeCount : summary.attention[key].length
|
||||
return (
|
||||
<button key={key} type="button" className={`fleet-attention__filter${activeAttention === key ? ' is-active' : ''}`}
|
||||
aria-pressed={activeAttention === key} onClick={() => onAttentionSelect(activeAttention === key ? null : key)}>
|
||||
{label} <strong>{count}</strong>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</aside>
|
||||
</>
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { timeAgo } from './nodeStatus'
|
||||
import ActionMenu from '../ActionMenu'
|
||||
|
||||
function SortButton({ column, label, sort, onSortChange }) {
|
||||
const active = sort.key === column
|
||||
const nextDirection = active && sort.direction === 'asc' ? 'desc' : 'asc'
|
||||
return (
|
||||
<button type="button" className="fleet-table__sort" onClick={() => onSortChange({ key: column, direction: nextDirection })}
|
||||
aria-label={`Sort by ${label.toLowerCase()}${active ? `, ${sort.direction}ending` : ''}`}>
|
||||
{label} {active && <i className={`fas fa-arrow-${sort.direction === 'asc' ? 'up' : 'down'}`} aria-hidden="true" />}
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
export default function ModelFleetTable({ models, selectedName, inspectorOpen, onInspect, onStop, stoppingName, sort, onSortChange }) {
|
||||
return (
|
||||
<div className="fleet-table-wrap model-fleet-table-wrap">
|
||||
<table className="fleet-table model-fleet-table" aria-label="Running models">
|
||||
<thead><tr>
|
||||
<th><SortButton column="model_name" label="Model" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th><SortButton column="replica_count" label="Replicas" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th><SortButton column="node_count" label="Nodes" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th><SortButton column="in_flight" label="In flight" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th>Backends</th>
|
||||
<th><SortButton column="last_used" label="Last used" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th className="model-fleet-table__actions"><span className="sr-only">Actions</span></th>
|
||||
</tr></thead>
|
||||
<tbody>{models.map(model => {
|
||||
const selected = selectedName === model.model_name
|
||||
const expanded = selected && inspectorOpen
|
||||
return (
|
||||
<tr key={model.model_name} className={`fleet-table__row${selected ? ' is-selected' : ''}`}>
|
||||
<td><button type="button" className="fleet-table__node" aria-label={`Inspect ${model.model_name}`}
|
||||
aria-pressed={selected} aria-expanded={expanded} aria-current={selected ? 'true' : undefined}
|
||||
aria-controls={expanded ? 'model-inspector' : undefined}
|
||||
onClick={event => onInspect(model, event.currentTarget)}>{model.model_name}</button></td>
|
||||
<td>{model.replica_count}</td>
|
||||
<td>{model.node_count}</td>
|
||||
<td>{model.in_flight}</td>
|
||||
<td><div className="model-backend-list">{model.backend_types.length ? model.backend_types.map(backend => <span key={backend}>{backend}</span>) : <span className="fleet-table__unknown">Unknown</span>}</div></td>
|
||||
<td>{model.last_used ? timeAgo(model.last_used) : <span className="fleet-table__unknown">Never</span>}</td>
|
||||
<td className="model-fleet-table__actions">
|
||||
<ActionMenu
|
||||
compact
|
||||
ariaLabel={`${model.model_name} actions`}
|
||||
triggerLabel={`Actions for ${model.model_name}`}
|
||||
items={[{
|
||||
key: 'stop',
|
||||
icon: 'fa-stop',
|
||||
label: stoppingName === model.model_name ? 'Stopping…' : 'Stop model…',
|
||||
danger: true,
|
||||
disabled: !!stoppingName,
|
||||
onClick: invoker => onStop(model, invoker),
|
||||
}]}
|
||||
/>
|
||||
</td>
|
||||
</tr>
|
||||
)
|
||||
})}</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import { useEffect, useRef } from 'react'
|
||||
import StatusPill from './StatusPill'
|
||||
import { timeAgo } from './nodeStatus'
|
||||
import useInspectorDrawer from './useInspectorDrawer'
|
||||
|
||||
function groupReplicasByNode(replicas, nodes) {
|
||||
const nodeById = new Map(nodes.map(node => [node.id, node]))
|
||||
const groups = new Map()
|
||||
replicas.forEach(replica => {
|
||||
const nodeId = String(replica?.node_id ?? '').trim()
|
||||
const key = nodeId || `unknown:${replica?.id ?? groups.size}`
|
||||
if (!groups.has(key)) groups.set(key, { nodeId, node: nodeById.get(nodeId) || null, replicas: [] })
|
||||
groups.get(key).replicas.push(replica)
|
||||
})
|
||||
return [...groups.values()]
|
||||
}
|
||||
|
||||
function latestUse(replicas) {
|
||||
const values = replicas.map(replica => ({ value: replica.last_used, time: Date.parse(replica.last_used) })).filter(entry => Number.isFinite(entry.time))
|
||||
return values.length ? values.sort((left, right) => right.time - left.time)[0].value : null
|
||||
}
|
||||
|
||||
export default function ModelInspector({ model, nodes, open, onClose, onOpenNode, focusNodeId }) {
|
||||
const closeRef = useRef(null)
|
||||
const drawerRef = useRef(null)
|
||||
const nodeButtonRefs = useRef(new Map())
|
||||
const modal = useInspectorDrawer(open, onClose, drawerRef)
|
||||
|
||||
useEffect(() => {
|
||||
if (open) closeRef.current?.focus()
|
||||
}, [open, model?.model_name])
|
||||
|
||||
useEffect(() => {
|
||||
if (open && focusNodeId) nodeButtonRefs.current.get(focusNodeId)?.focus()
|
||||
}, [open, focusNodeId])
|
||||
|
||||
if (!open || !model) return null
|
||||
const nodeGroups = groupReplicasByNode(model.replicas, nodes)
|
||||
|
||||
return <>
|
||||
<div className="node-inspector__scrim" aria-hidden="true" onClick={onClose} />
|
||||
<aside ref={drawerRef} id="model-inspector" className="node-inspector model-inspector" aria-label="Model inspector" role={modal ? 'dialog' : undefined} aria-modal={modal ? 'true' : undefined} tabIndex={modal ? -1 : undefined}>
|
||||
<header className="node-inspector__header">
|
||||
<div className="node-inspector__topbar"><span className="fleet-kicker">Running model</span><button ref={closeRef} type="button" className="btn btn-ghost btn-sm" aria-label="Close model inspector" onClick={onClose}><i className="fas fa-times" /></button></div>
|
||||
<h2>{model.model_name}</h2>
|
||||
<p>Replica placement across the active fleet</p>
|
||||
</header>
|
||||
<div className="node-inspector__body">
|
||||
<section className="node-inspector__section model-inspector__summary">
|
||||
<h3>Model summary</h3>
|
||||
<dl className="node-inspector__metrics">
|
||||
<div><dt>Replicas</dt><dd>{model.replica_count}</dd></div>
|
||||
<div><dt>Nodes</dt><dd>{model.node_count}</dd></div>
|
||||
<div><dt>In-flight work</dt><dd>{model.in_flight}</dd></div>
|
||||
<div><dt>Last used</dt><dd>{model.last_used ? timeAgo(model.last_used) : 'Never'}</dd></div>
|
||||
</dl>
|
||||
<div className="model-backend-list model-inspector__backends" aria-label="Backend types">{model.backend_types.length ? model.backend_types.map(backend => <span key={backend}>{backend}</span>) : <span className="text-muted">Backend unknown</span>}</div>
|
||||
</section>
|
||||
<section className="node-inspector__section">
|
||||
<h3>Replica placement</h3>
|
||||
<div className="model-inspector__nodes">{nodeGroups.map(group => {
|
||||
const inFlight = group.replicas.reduce((total, replica) => total + (Number.isFinite(replica.in_flight) ? Math.max(0, Math.floor(replica.in_flight)) : 0), 0)
|
||||
const lastUsed = latestUse(group.replicas)
|
||||
const name = group.node?.name || group.nodeId || 'Unknown node'
|
||||
return (
|
||||
<article className="model-inspector__node" key={group.nodeId || group.replicas[0]?.id}>
|
||||
<div className="model-inspector__node-heading">
|
||||
{group.node
|
||||
? <button ref={element => { if (element) nodeButtonRefs.current.set(group.nodeId, element); else nodeButtonRefs.current.delete(group.nodeId) }}
|
||||
type="button" onClick={event => onOpenNode(group.node, event.currentTarget)} aria-label={`Open node ${name}`}>{name}</button>
|
||||
: <strong>{name}</strong>}
|
||||
{group.node ? <StatusPill status={group.node.status} /> : <span className="status-pill status-pill--neutral">Unknown</span>}
|
||||
</div>
|
||||
<p>{group.replicas.length} replica{group.replicas.length === 1 ? '' : 's'} · {inFlight} in flight · {lastUsed ? timeAgo(lastUsed) : 'never used'}</p>
|
||||
<ul>{group.replicas.map(replica => <li key={replica.id || `${replica.replica_index}:${replica.address}`}><span>Replica {Number.isFinite(replica.replica_index) ? replica.replica_index + 1 : '—'}</span><code>{replica.address || 'No address'}</code></li>)}</ul>
|
||||
</article>
|
||||
)
|
||||
})}</div>
|
||||
</section>
|
||||
</div>
|
||||
<footer className="node-inspector__actions node-inspector__actions--single"><button type="button" className="btn btn-secondary btn-sm" onClick={onClose}>Close</button></footer>
|
||||
</aside>
|
||||
</>
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import StatusPill from './StatusPill'
|
||||
import { formatBytes, timeAgo } from './nodeStatus'
|
||||
import { capacityReading, groupNodes, nodeLifecycleAction } from '../../utils/nodeFleet'
|
||||
|
||||
function MetricCell({ total, available, tone }) {
|
||||
const reading = capacityReading(total, available)
|
||||
if (!reading) return <span className="fleet-table__unknown">No data</span>
|
||||
const percent = Math.round(reading.usagePercent)
|
||||
return (
|
||||
<div className={`fleet-table__resource fleet-table__resource--${tone}`} aria-label={`${formatBytes(reading.used)} of ${formatBytes(reading.total)} used`}>
|
||||
<progress className="fleet-table__resource-track" max="100" value={percent} aria-hidden="true" />
|
||||
<span>{formatBytes(reading.used)} / {formatBytes(reading.total)}</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function CapacityCell({ node }) {
|
||||
return <div className="fleet-table__capacity">
|
||||
<div className="fleet-table__capacity-row"><b>VRAM</b><MetricCell total={node.total_vram} available={node.available_vram} tone="vram" /></div>
|
||||
<div className="fleet-table__capacity-row"><b>RAM</b><MetricCell total={node.total_ram} available={node.available_ram} tone="ram" /></div>
|
||||
</div>
|
||||
}
|
||||
|
||||
function SortButton({ column, label, sort, onSortChange }) {
|
||||
const active = sort.key === column
|
||||
const nextDirection = active && sort.direction === 'asc' ? 'desc' : 'asc'
|
||||
return (
|
||||
<button type="button" className="fleet-table__sort" onClick={() => onSortChange({ key: column, direction: nextDirection })}
|
||||
aria-label={`Sort by ${label.toLowerCase()}${active ? `, ${sort.direction}ending` : ''}`}>
|
||||
{label} {active && <i className={`fas fa-arrow-${sort.direction === 'asc' ? 'up' : 'down'}`} aria-hidden="true" />}
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
export default function NodeFleetTable({ nodes, selectedIds, onSelectionChange, onInspect, sort, onSortChange, groupBy, onApprove }) {
|
||||
const groups = groupNodes(nodes, groupBy)
|
||||
const visibleIds = nodes.map(node => node.id)
|
||||
const selectedVisible = visibleIds.filter(id => selectedIds.has(id)).length
|
||||
const setMany = (ids, selected) => {
|
||||
const next = new Set(selectedIds)
|
||||
ids.forEach(id => selected ? next.add(id) : next.delete(id))
|
||||
onSelectionChange(next)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="fleet-table-wrap">
|
||||
<table className="fleet-table" aria-label="Fleet nodes">
|
||||
<thead>
|
||||
<tr>
|
||||
<th className="fleet-table__check"><input type="checkbox" aria-label="Select visible nodes" checked={visibleIds.length > 0 && selectedVisible === visibleIds.length}
|
||||
ref={input => { if (input) input.indeterminate = selectedVisible > 0 && selectedVisible < visibleIds.length }}
|
||||
onChange={event => setMany(visibleIds, event.target.checked)} /></th>
|
||||
<th><SortButton column="name" label="Node" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th><SortButton column="status" label="Status" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th>Capacity</th><th>CPU</th>
|
||||
<th><SortButton column="model_count" label="Workload" sort={sort} onSortChange={onSortChange} /></th>
|
||||
<th>Heartbeat</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{groups.flatMap(group => {
|
||||
const groupIds = group.nodes.map(node => node.id)
|
||||
const groupSelected = groupIds.filter(id => selectedIds.has(id)).length
|
||||
const rows = group.nodes.map(node => (
|
||||
<tr key={node.id} className={`fleet-table__row${selectedIds.has(node.id) ? ' is-selected' : ''}`} tabIndex="0" onClick={event => onInspect(node, event.currentTarget)}
|
||||
onKeyDown={event => { if (event.target === event.currentTarget && (event.key === 'Enter' || event.key === ' ')) { event.preventDefault(); onInspect(node, event.currentTarget) } }}>
|
||||
<td onClick={event => event.stopPropagation()}><input type="checkbox" aria-label={`Select ${node.name}`} checked={selectedIds.has(node.id)}
|
||||
onChange={event => setMany([node.id], event.target.checked)} /></td>
|
||||
<td><button type="button" className="fleet-table__node" aria-label={`Inspect ${node.name}`} onClick={event => { event.stopPropagation(); onInspect(node, event.currentTarget) }}>{node.name}</button><span>{node.node_type || 'backend'} · {node.address || 'No address'}</span></td>
|
||||
<td><StatusPill status={node.status} />{nodeLifecycleAction(node.status) === 'approve' && <button type="button" className="fleet-table__approve" aria-label={`Approve ${node.name}`} onClick={event => { event.stopPropagation(); onApprove(node.id) }}>Approve</button>}</td>
|
||||
<td><CapacityCell node={node} /></td>
|
||||
<td>{node.cpu_logical_cores > 0 && Number.isFinite(node.cpu_usage_percent) ? `${node.cpu_usage_percent.toFixed(0)}% · ${node.cpu_logical_cores}c` : <span className="fleet-table__unknown">No data</span>}</td>
|
||||
<td><strong>{node.model_count ?? 0} models</strong><span className="fleet-table__subvalue">{node.in_flight_count ?? 0} in flight</span></td>
|
||||
<td>{timeAgo(node.last_heartbeat)}</td>
|
||||
</tr>
|
||||
))
|
||||
if (groupBy === 'none') return rows
|
||||
return [
|
||||
<tr key={`group:${group.key}`} className="fleet-table__group">
|
||||
<th colSpan="7"><label><input type="checkbox" aria-label={`Select ${group.label} group`} checked={groupIds.length > 0 && groupSelected === groupIds.length}
|
||||
ref={input => { if (input) input.indeterminate = groupSelected > 0 && groupSelected < groupIds.length }}
|
||||
onChange={event => setMany(groupIds, event.target.checked)} /> {group.label}</label><span>{group.nodes.length} nodes · {groupSelected} selected</span></th>
|
||||
</tr>,
|
||||
...rows,
|
||||
]
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
{nodes.length === 0 && <div className="fleet-table__empty">No nodes match the current view.</div>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import StatusPill from './StatusPill'
|
||||
import { formatBytes, formatCapacity, timeAgo } from './nodeStatus'
|
||||
import { nodesApi } from '../../utils/api'
|
||||
import { capacityReading, nodeLifecycleAction } from '../../utils/nodeFleet'
|
||||
import useInspectorDrawer from './useInspectorDrawer'
|
||||
|
||||
function InspectorMetric({ label, children }) {
|
||||
return <div><dt>{label}</dt><dd>{children}</dd></div>
|
||||
}
|
||||
|
||||
function ResourceBar({ label, total, available, tone }) {
|
||||
const reading = capacityReading(total, available)
|
||||
if (!reading) return <InspectorMetric label={label}>No data</InspectorMetric>
|
||||
const percent = Math.round(reading.usagePercent)
|
||||
return (
|
||||
<div className={`node-inspector__resource node-inspector__resource--${tone}`}>
|
||||
<div className="node-inspector__resource-label"><strong>{label} <span>{formatBytes(reading.used)} / {formatBytes(reading.total)}</span></strong><span>{formatBytes(reading.available)} free</span></div>
|
||||
<progress className="node-inspector__resource-track" max="100" value={percent} aria-label={`${label} usage`} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default function NodeInspector({ node, open, onClose, onApprove, onDrain, onResume, onBack, backLabel }) {
|
||||
const [backends, setBackends] = useState(null)
|
||||
const [backendError, setBackendError] = useState('')
|
||||
const nodeId = node?.id
|
||||
const backRef = useRef(null)
|
||||
const closeRef = useRef(null)
|
||||
const drawerRef = useRef(null)
|
||||
const hasBack = Boolean(onBack)
|
||||
const modal = useInspectorDrawer(open, onClose, drawerRef)
|
||||
|
||||
useEffect(() => {
|
||||
if (open) (hasBack ? backRef : closeRef).current?.focus()
|
||||
}, [open, nodeId, hasBack])
|
||||
|
||||
useEffect(() => {
|
||||
if (!open || !nodeId) return undefined
|
||||
let current = true
|
||||
setBackends(null)
|
||||
setBackendError('')
|
||||
nodesApi.getBackends(nodeId).then(data => {
|
||||
if (current) setBackends(Array.isArray(data) ? data : [])
|
||||
}).catch(error => {
|
||||
if (current) setBackendError(error.message || 'Unable to load backends')
|
||||
})
|
||||
return () => { current = false }
|
||||
}, [open, nodeId])
|
||||
|
||||
if (!open || !node) return null
|
||||
const cpuKnown = node.cpu_logical_cores > 0 && Number.isFinite(node.cpu_usage_percent) && Number.isFinite(node.cpu_load_1)
|
||||
const disk = capacityReading(node.total_disk, node.available_disk)
|
||||
const lifecycleAction = nodeLifecycleAction(node.status)
|
||||
|
||||
return <>
|
||||
<div className="node-inspector__scrim" aria-hidden="true" onClick={onClose} />
|
||||
<aside ref={drawerRef} className="node-inspector" aria-label="Node inspector" role={modal ? 'dialog' : undefined} aria-modal={modal ? 'true' : undefined} tabIndex={modal ? -1 : undefined}>
|
||||
<header className="node-inspector__header">
|
||||
<div className="node-inspector__topbar">
|
||||
{onBack
|
||||
? <button ref={backRef} type="button" className="node-inspector__back" onClick={onBack}><i className="fas fa-arrow-left" aria-hidden="true" /> {backLabel || 'Back'}</button>
|
||||
: <span className="fleet-kicker">Node inspector</span>}
|
||||
<button ref={closeRef} type="button" className="btn btn-ghost btn-sm" aria-label="Close node inspector" onClick={onClose}><i className="fas fa-times" /></button>
|
||||
</div>
|
||||
<div className="node-inspector__identity">
|
||||
<h2>{node.name}</h2>
|
||||
<p>{node.labels?.zone ? `${node.labels.zone} · ` : ''}{node.node_type || 'worker'} node</p>
|
||||
<StatusPill status={node.status} />
|
||||
</div>
|
||||
</header>
|
||||
<div className="node-inspector__body">
|
||||
<section className="node-inspector__section">
|
||||
<h3>Node</h3>
|
||||
<dl className="node-inspector__metrics">
|
||||
<InspectorMetric label="Address"><span className="node-inspector__address">{node.address || 'No address reported'}</span></InspectorMetric>
|
||||
<InspectorMetric label="Heartbeat">{timeAgo(node.last_heartbeat)}</InspectorMetric>
|
||||
</dl>
|
||||
<div className="node-inspector__labels" aria-label="Node labels">{Object.keys(node.labels || {}).length ? Object.entries(node.labels).map(([key, value]) => <span key={key}>{key}={value}</span>) : <span className="text-muted">No labels</span>}</div>
|
||||
</section>
|
||||
<section className="node-inspector__section">
|
||||
<h3>Resources</h3>
|
||||
<ResourceBar label="VRAM" total={node.total_vram} available={node.available_vram} tone="vram" />
|
||||
<ResourceBar label="RAM" total={node.total_ram} available={node.available_ram} tone="ram" />
|
||||
<dl className="node-inspector__metrics">
|
||||
<InspectorMetric label="CPU">{cpuKnown ? `${node.cpu_usage_percent.toFixed(1)}% of ${node.cpu_logical_cores} cores · ${node.cpu_load_1.toFixed(2)} load` : 'No data'}</InspectorMetric>
|
||||
<InspectorMetric label="Models disk">{disk ? <>{formatCapacity(disk.used, disk.total)} · {formatBytes(disk.available)} free</> : 'No data'}</InspectorMetric>
|
||||
</dl>
|
||||
</section>
|
||||
<section className="node-inspector__section">
|
||||
<h3>Workload</h3>
|
||||
<dl className="node-inspector__metrics">
|
||||
<InspectorMetric label="Loaded models">{node.model_count ?? 0}</InspectorMetric>
|
||||
<InspectorMetric label="Backends">{backendError ? <span className="text-error">{backendError}</span> : backends === null ? 'Loading…' : `${backends.length} backend${backends.length === 1 ? '' : 's'}`}</InspectorMetric>
|
||||
<InspectorMetric label="In-flight work">{node.in_flight_count ?? 0}</InspectorMetric>
|
||||
</dl>
|
||||
</section>
|
||||
</div>
|
||||
<footer className="node-inspector__actions">
|
||||
<a className="btn btn-primary btn-sm" href={`/app/nodes/${encodeURIComponent(node.id)}`} aria-label="Open full node details">Open full details</a>
|
||||
{lifecycleAction === 'approve' && <button type="button" className="btn btn-primary btn-sm" onClick={() => onApprove(node.id)}>Approve</button>}
|
||||
{lifecycleAction === 'resume' && <button type="button" className="btn btn-secondary btn-sm" onClick={() => onResume(node.id)}>Resume</button>}
|
||||
{lifecycleAction === 'drain' && <button type="button" className="btn btn-secondary btn-sm" onClick={() => onDrain(node.id)}>Drain</button>}
|
||||
</footer>
|
||||
</aside>
|
||||
</>
|
||||
}
|
||||
@@ -21,6 +21,25 @@ export function formatVRAM(bytes) {
|
||||
return gb >= 1 ? `${gb.toFixed(1)} GB` : `${(bytes / (1024 * 1024)).toFixed(0)} MB`
|
||||
}
|
||||
|
||||
export function formatBytes(bytes) {
|
||||
if (typeof bytes !== 'number' || !Number.isFinite(bytes) || bytes < 0) return 'No data'
|
||||
if (bytes < 1024) return `${Math.round(bytes)} B`
|
||||
const units = ['KB', 'MB', 'GB', 'TB', 'PB']
|
||||
let value = bytes / 1024
|
||||
let unit = units[0]
|
||||
for (let index = 1; index < units.length && value >= 1024; index += 1) {
|
||||
value /= 1024
|
||||
unit = units[index]
|
||||
}
|
||||
const precision = value >= 10 ? 0 : 1
|
||||
return `${value.toFixed(precision).replace(/\.0$/, '')} ${unit}`
|
||||
}
|
||||
|
||||
export function formatCapacity(used, total) {
|
||||
if (!(total > 0)) return 'No data'
|
||||
return `${formatBytes(used)} / ${formatBytes(total)}`
|
||||
}
|
||||
|
||||
export function timeAgo(dateString) {
|
||||
if (!dateString) return 'never'
|
||||
const seconds = Math.floor((Date.now() - new Date(dateString).getTime()) / 1000)
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
|
||||
const MOBILE_DRAWER_QUERY = '(max-width: 768px)'
|
||||
const FOCUSABLE = 'button:not(:disabled), [href], input:not(:disabled), select:not(:disabled), textarea:not(:disabled), [tabindex]:not([tabindex="-1"])'
|
||||
|
||||
function isolateBackground(drawer) {
|
||||
const changed = new Map()
|
||||
let current = drawer
|
||||
|
||||
while (current?.parentElement && current.parentElement !== document.documentElement) {
|
||||
const parent = current.parentElement
|
||||
for (const sibling of parent.children) {
|
||||
if (sibling === current || sibling.classList.contains('node-inspector__scrim') || changed.has(sibling)) continue
|
||||
changed.set(sibling, {
|
||||
ariaHidden: sibling.getAttribute('aria-hidden'),
|
||||
inert: sibling.hasAttribute('inert'),
|
||||
})
|
||||
sibling.setAttribute('aria-hidden', 'true')
|
||||
sibling.setAttribute('inert', '')
|
||||
}
|
||||
current = parent
|
||||
}
|
||||
|
||||
return () => {
|
||||
for (const [element, previous] of changed) {
|
||||
if (previous.ariaHidden === null) element.removeAttribute('aria-hidden')
|
||||
else element.setAttribute('aria-hidden', previous.ariaHidden)
|
||||
if (!previous.inert) element.removeAttribute('inert')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default function useInspectorDrawer(open, onClose, drawerRef) {
|
||||
const onCloseRef = useRef(onClose)
|
||||
const [modal, setModal] = useState(() => typeof window !== 'undefined' && window.matchMedia(MOBILE_DRAWER_QUERY).matches)
|
||||
|
||||
useEffect(() => { onCloseRef.current = onClose }, [onClose])
|
||||
|
||||
useEffect(() => {
|
||||
const media = window.matchMedia(MOBILE_DRAWER_QUERY)
|
||||
const update = event => setModal(event.matches)
|
||||
setModal(media.matches)
|
||||
media.addEventListener('change', update)
|
||||
return () => media.removeEventListener('change', update)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return undefined
|
||||
const drawer = drawerRef.current
|
||||
if (!drawer) return undefined
|
||||
const previousOverflow = document.body.style.overflow
|
||||
const restoreBackground = modal ? isolateBackground(drawer) : () => {}
|
||||
if (modal) document.body.style.overflow = 'hidden'
|
||||
|
||||
const handleKeyDown = event => {
|
||||
const blockingModal = [...document.querySelectorAll('[aria-modal="true"]')].some(element => element !== drawer)
|
||||
if (event.defaultPrevented || blockingModal) return
|
||||
if (event.key === 'Escape') {
|
||||
event.preventDefault()
|
||||
onCloseRef.current()
|
||||
return
|
||||
}
|
||||
if (!modal || event.key !== 'Tab') return
|
||||
const focusable = [...drawer.querySelectorAll(FOCUSABLE)].filter(element => element.getClientRects().length > 0)
|
||||
if (!focusable.length) {
|
||||
event.preventDefault()
|
||||
drawer.focus()
|
||||
return
|
||||
}
|
||||
const first = focusable[0]
|
||||
const last = focusable[focusable.length - 1]
|
||||
if (event.shiftKey && (document.activeElement === first || !drawer.contains(document.activeElement))) {
|
||||
event.preventDefault()
|
||||
last.focus()
|
||||
} else if (!event.shiftKey && (document.activeElement === last || !drawer.contains(document.activeElement))) {
|
||||
event.preventDefault()
|
||||
first.focus()
|
||||
}
|
||||
}
|
||||
|
||||
document.addEventListener('keydown', handleKeyDown)
|
||||
return () => {
|
||||
document.removeEventListener('keydown', handleKeyDown)
|
||||
restoreBackground()
|
||||
if (modal) document.body.style.overflow = previousOverflow
|
||||
}
|
||||
}, [drawerRef, modal, open])
|
||||
|
||||
return modal
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import StatusPill from '../components/nodes/StatusPill'
|
||||
import CapacityEditor from '../components/nodes/CapacityEditor'
|
||||
import KeyValueChips from '../components/nodes/KeyValueChips'
|
||||
import { formatVRAM, modelStateConfig, timeAgo } from '../components/nodes/nodeStatus'
|
||||
import { capacityReading, nodeLifecycleAction } from '../utils/nodeFleet'
|
||||
|
||||
// Deep-linkable node management home. Reached by clicking a roster panel on
|
||||
// /app/nodes. Surfaces what's running here plus the management affordances
|
||||
@@ -53,6 +54,7 @@ export default function NodeDetail() {
|
||||
|
||||
const drain = async () => { try { await nodesApi.drain(id); addToast('Node set to draining', 'success'); refresh() } catch (e) { addToast(e.message, 'error') } }
|
||||
const resume = async () => { try { await nodesApi.resume(id); addToast('Node resumed', 'success'); refresh() } catch (e) { addToast(e.message, 'error') } }
|
||||
const approve = async () => { try { await nodesApi.approve(id); addToast('Node approved', 'success'); refresh() } catch (e) { addToast(e.message, 'error') } }
|
||||
const remove = async () => { try { await nodesApi.delete(id); addToast('Node removed', 'success'); navigate('/app/nodes') } catch (e) { addToast(e.message, 'error') } }
|
||||
const unload = async (name) => { try { await nodesApi.unloadModel(id, name); addToast(`Model "${name}" unloaded`, 'success'); refresh() } catch (e) { addToast(e.message, 'error') } }
|
||||
// The upgrade runs async via the gallery job queue (202 + jobID); the
|
||||
@@ -63,8 +65,10 @@ export default function NodeDetail() {
|
||||
const addLabel = async (k, v) => { try { await nodesApi.mergeLabels(id, { [k]: v }); refresh() } catch (e) { addToast(e.message, 'error') } }
|
||||
const delLabel = async (k) => { try { await nodesApi.deleteLabel(id, k); refresh() } catch (e) { addToast(e.message, 'error') } }
|
||||
|
||||
const usedVRAM = node.total_vram && node.available_vram != null ? node.total_vram - node.available_vram : 0
|
||||
const usedRAM = node.total_ram && node.available_ram != null ? node.total_ram - node.available_ram : 0
|
||||
const vram = capacityReading(node.total_vram, node.available_vram)
|
||||
const ram = capacityReading(node.total_ram, node.available_ram)
|
||||
const disk = capacityReading(node.total_disk, node.available_disk)
|
||||
const lifecycleAction = nodeLifecycleAction(node.status)
|
||||
// {modelName: replicaCount} of loaded models so the shrink confirm can warn
|
||||
// if the new cap is below the actual count of any single model on this node.
|
||||
const loadedModelCounts = (() => {
|
||||
@@ -81,9 +85,9 @@ export default function NodeDetail() {
|
||||
supporting={node.address}
|
||||
actions={
|
||||
<>
|
||||
{node.status === 'draining'
|
||||
? <button className="btn btn-secondary btn-sm" onClick={resume}><i className="fas fa-play" /> Resume</button>
|
||||
: <button className="btn btn-secondary btn-sm" onClick={drain}><i className="fas fa-pause" /> Drain</button>}
|
||||
{lifecycleAction === 'approve' && <button className="btn btn-primary btn-sm" onClick={approve}><i className="fas fa-check" /> Approve</button>}
|
||||
{lifecycleAction === 'resume' && <button className="btn btn-secondary btn-sm" onClick={resume}><i className="fas fa-play" /> Resume</button>}
|
||||
{lifecycleAction === 'drain' && <button className="btn btn-secondary btn-sm" onClick={drain}><i className="fas fa-pause" /> Drain</button>}
|
||||
<button className="btn btn-danger btn-sm" onClick={() => setConfirmRemove(true)}><i className="fas fa-trash" /> Remove</button>
|
||||
</>
|
||||
}
|
||||
@@ -91,26 +95,27 @@ export default function NodeDetail() {
|
||||
|
||||
{/* Inline resource and activity metrics - no boxes, just labelled values. */}
|
||||
<div className="node-detail__metrics">
|
||||
{node.total_vram > 0 && (
|
||||
<div>
|
||||
<div className="drawer-eyebrow">VRAM</div>
|
||||
<span className="cell-mono">{formatVRAM(usedVRAM) || '0'} / {formatVRAM(node.total_vram)}</span>
|
||||
</div>
|
||||
)}
|
||||
{node.total_ram > 0 && (
|
||||
<div>
|
||||
<div className="drawer-eyebrow">RAM</div>
|
||||
<span className="cell-mono">{formatVRAM(usedRAM) || '0'} / {formatVRAM(node.total_ram)}</span>
|
||||
</div>
|
||||
)}
|
||||
{node.total_disk > 0 && (
|
||||
<div>
|
||||
<div>
|
||||
<div className="drawer-eyebrow">VRAM</div>
|
||||
<span className="cell-mono">{vram ? `${formatVRAM(vram.used) || '0'} / ${formatVRAM(vram.total)}` : 'No data'}</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="drawer-eyebrow">RAM</div>
|
||||
<span className="cell-mono">{ram ? `${formatVRAM(ram.used) || '0'} / ${formatVRAM(ram.total)}` : 'No data'}</span>
|
||||
</div>
|
||||
<div>
|
||||
{/* Free space on the worker's MODELS filesystem. A node can look
|
||||
perfectly healthy on VRAM while having nowhere to put the
|
||||
weights, which is why this sits next to VRAM rather than
|
||||
buried in a diagnostics panel. */}
|
||||
<div className="drawer-eyebrow">Models disk free</div>
|
||||
<span className="cell-mono">{formatVRAM(node.available_disk || 0) || '0'} / {formatVRAM(node.total_disk)}</span>
|
||||
<div className="drawer-eyebrow">Models disk free</div>
|
||||
<span className="cell-mono">{disk ? `${formatVRAM(disk.available) || '0'} / ${formatVRAM(disk.total)}` : 'No data'}</span>
|
||||
</div>
|
||||
{node.cpu_logical_cores > 0 && Number.isFinite(node.cpu_usage_percent) && (
|
||||
<div>
|
||||
<div className="drawer-eyebrow">CPU</div>
|
||||
<span className="cell-mono">{node.cpu_usage_percent.toFixed(1)}% of {node.cpu_logical_cores} cores</span>
|
||||
{Number.isFinite(node.cpu_load_1) && <span className="node-detail__metric-note">{node.cpu_load_1.toFixed(2)} load (1m)</span>}
|
||||
</div>
|
||||
)}
|
||||
<div>
|
||||
@@ -169,7 +174,6 @@ export default function NodeDetail() {
|
||||
{m.model_name}
|
||||
{showReplica && (
|
||||
<span
|
||||
className="cell-mono"
|
||||
aria-label={`replica ${m.replica_index ?? 0}`}
|
||||
title={`Replica ${m.replica_index ?? 0} on this node`}
|
||||
className="inline-tag"
|
||||
|
||||
@@ -1,347 +1,391 @@
|
||||
import { useState, useEffect, useCallback, useMemo } from 'react'
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useOutletContext } from 'react-router-dom'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { nodesApi } from '../utils/api'
|
||||
import { backendControlApi, nodesApi } from '../utils/api'
|
||||
import { filterModels, filterNodes, groupModels, paginateModels, paginateNodes, runBounded, sortModels, sortNodes, summarizeFleet } from '../utils/nodeFleet'
|
||||
import LoadingSpinner from '../components/LoadingSpinner'
|
||||
import PageHeader from '../components/PageHeader'
|
||||
import ConfirmDialog from '../components/ConfirmDialog'
|
||||
import ImageSelector, { useImageSelector, dockerImage, dockerFlags } from '../components/ImageSelector'
|
||||
import ClusterPulse from '../components/nodes/ClusterPulse'
|
||||
import AttentionCallout from '../components/nodes/AttentionCallout'
|
||||
import NodePanel from '../components/nodes/NodePanel'
|
||||
|
||||
|
||||
function StepNumber({ n, bg, color }) {
|
||||
return (
|
||||
<span className="p2p-step" style={{ background: bg, color }}>{n}</span>
|
||||
)
|
||||
}
|
||||
import ClusterOverview from '../components/nodes/ClusterOverview'
|
||||
import NodeFleetTable from '../components/nodes/NodeFleetTable'
|
||||
import NodeInspector from '../components/nodes/NodeInspector'
|
||||
import ModelFleetTable from '../components/nodes/ModelFleetTable'
|
||||
import ModelInspector from '../components/nodes/ModelInspector'
|
||||
import ImageSelector, { dockerFlags, dockerImage, useImageSelector } from '../components/ImageSelector'
|
||||
|
||||
function CommandBlock({ command, addToast }) {
|
||||
const copy = () => {
|
||||
navigator.clipboard.writeText(command)
|
||||
addToast('Copied to clipboard', 'success', 2000)
|
||||
}
|
||||
return <div className="p2p-cmd"><pre>{command}</pre><button onClick={copy} className="btn btn-sm p2p-cmd__copy" title="Copy"><i className="fas fa-copy" /></button></div>
|
||||
}
|
||||
|
||||
function WorkerHintCard({ addToast, nodeType = 'backend', hasWorkers }) {
|
||||
const frontendUrl = window.location.origin
|
||||
const { selected, setSelected, option, dev, setDev } = useImageSelector('cpu')
|
||||
const isAgent = nodeType === 'agent'
|
||||
const workerCmd = isAgent ? 'agent-worker' : 'worker'
|
||||
const flags = dockerFlags(option)
|
||||
const flagsString = flags ? `${flags} \
|
||||
` : ''
|
||||
return (
|
||||
<div className="p2p-cmd">
|
||||
<pre>{command}</pre>
|
||||
<button
|
||||
onClick={copy}
|
||||
className="btn btn-sm p2p-cmd__copy"
|
||||
title="Copy"
|
||||
>
|
||||
<i className="fas fa-copy" />
|
||||
</button>
|
||||
<div className="card pad-lg mb-xl">
|
||||
<h3 className="panel-title"><i className={`fas ${hasWorkers ? 'fa-plus-circle' : 'fa-info-circle'} text-primary`} />{hasWorkers ? 'Register another worker' : 'No workers registered yet'}</h3>
|
||||
<p className="text-base text-secondary mb-md">Start a worker to add compute capacity. It will register with this frontend and appear here automatically.</p>
|
||||
<p className="form-label">Select your hardware</p>
|
||||
<ImageSelector selected={selected} onSelect={setSelected} dev={dev} onDevChange={setDev} />
|
||||
<div className="stack">
|
||||
<div><p className="form-label">CLI</p><CommandBlock command={`local-ai ${workerCmd} \
|
||||
--register-to "${frontendUrl}" \
|
||||
--nats-url "nats://nats:4222" \
|
||||
--registration-token "$LOCALAI_REGISTRATION_TOKEN"`} addToast={addToast} /></div>
|
||||
<div><p className="form-label">Docker</p><CommandBlock command={`docker run --net host ${flagsString}\
|
||||
-e LOCALAI_REGISTER_TO="${frontendUrl}" \
|
||||
-e LOCALAI_NATS_URL="nats://nats:4222" \
|
||||
-e LOCALAI_REGISTRATION_TOKEN="$TOKEN" \
|
||||
${dockerImage(option, dev)} ${workerCmd}`} addToast={addToast} /></div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function WorkerHintCard({ addToast, activeTab, hasWorkers }) {
|
||||
const frontendUrl = window.location.origin
|
||||
const { selected, setSelected, option, dev, setDev } = useImageSelector('cpu')
|
||||
const isAgent = activeTab === 'agent'
|
||||
const workerCmd = isAgent ? 'agent-worker' : 'worker'
|
||||
const flags = dockerFlags(option)
|
||||
const flagsStr = flags ? `${flags} \\\n ` : ''
|
||||
|
||||
const title = hasWorkers
|
||||
? (isAgent ? 'Add more agent workers' : 'Add more workers')
|
||||
: (isAgent ? 'No agent workers registered yet' : 'No workers registered yet')
|
||||
|
||||
function DisabledState({ addToast }) {
|
||||
return (
|
||||
<div className="card pad-lg mb-xl">
|
||||
<h3 className="panel-title">
|
||||
<i className={`fas ${hasWorkers ? 'fa-plus-circle' : 'fa-info-circle'} text-primary`} />
|
||||
{title}
|
||||
</h3>
|
||||
<p className="text-base text-secondary mb-md">
|
||||
{isAgent
|
||||
? 'Start agent worker nodes to execute MCP tools and agent tasks. Agent workers self-register with this frontend.'
|
||||
: 'Start worker nodes to scale inference across multiple machines. Workers self-register with this frontend.'}
|
||||
</p>
|
||||
|
||||
<p className="form-label">Select your hardware</p>
|
||||
<ImageSelector selected={selected} onSelect={setSelected} dev={dev} onDevChange={setDev} />
|
||||
|
||||
<div className="stack">
|
||||
<div>
|
||||
<p className="form-label">CLI</p>
|
||||
<CommandBlock
|
||||
command={`local-ai ${workerCmd} \\\n --register-to "${frontendUrl}" \\\n --nats-url "nats://nats:4222" \\\n --registration-token "$LOCALAI_REGISTRATION_TOKEN"`}
|
||||
addToast={addToast}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<p className="form-label">Docker</p>
|
||||
<CommandBlock
|
||||
command={`docker run --net host ${flagsStr}\\\n -e LOCALAI_REGISTER_TO="${frontendUrl}" \\\n -e LOCALAI_NATS_URL="nats://nats:4222" \\\n -e LOCALAI_REGISTRATION_TOKEN="$TOKEN" \\\n ${dockerImage(option, dev)} ${workerCmd}`}
|
||||
addToast={addToast}
|
||||
/>
|
||||
</div>
|
||||
<div className="page page--wide">
|
||||
<div className="p2p-hero"><i className="fas fa-network-wired" /><h1>Distributed Mode Not Enabled</h1><p>Enable distributed mode to manage backend nodes across multiple machines and route inference across the fleet.</p></div>
|
||||
<div className="card p2p-enable pad-lg">
|
||||
<h3 className="panel-title"><i className="fas fa-rocket text-accent" />How to Enable Distributed Mode</h3>
|
||||
<p className="form-label">Start LocalAI with distributed mode</p>
|
||||
<CommandBlock command={'local-ai run --distributed \\\n --distributed-db "postgres://user:pass@host/db" \\\n --distributed-nats "nats://host:4222"'} addToast={addToast} />
|
||||
<p className="text-note mt-md">Then register a worker and refresh this page. See the <a href="https://localai.io/features/distributed-mode/" target="_blank" rel="noopener noreferrer" className="text-primary">Distributed Mode documentation</a> for production setup.</p>
|
||||
</div>
|
||||
|
||||
<p className="text-note mt-md">
|
||||
For full setup instructions, architecture details, and Kubernetes deployment, see the{' '}
|
||||
<a href="https://localai.io/features/distributed-mode/" target="_blank" rel="noopener noreferrer"
|
||||
className="text-primary">Distributed Mode documentation <i className="fas fa-external-link-alt text-xs" /></a>.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function FleetSelect({ label, value, onChange, children }) {
|
||||
return (
|
||||
<label className="fleet-select-wrap">
|
||||
<span className="sr-only">{label}</span>
|
||||
<select className="fleet-select" aria-label={label} value={value} onChange={onChange}>{children}</select>
|
||||
<i className="fas fa-chevron-down fleet-select__chevron" aria-hidden="true" />
|
||||
</label>
|
||||
)
|
||||
}
|
||||
|
||||
export default function Nodes() {
|
||||
const { addToast } = useOutletContext()
|
||||
const { t } = useTranslation('admin')
|
||||
const [nodesList, setNodesList] = useState([])
|
||||
const [allModels, setAllModels] = useState([])
|
||||
const [nodes, setNodes] = useState([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [enabled, setEnabled] = useState(true)
|
||||
const [confirmDelete, setConfirmDelete] = useState(null)
|
||||
const [query, setQuery] = useState('')
|
||||
const [status, setStatus] = useState('')
|
||||
const [type, setType] = useState('')
|
||||
const [groupBy, setGroupBy] = useState('none')
|
||||
const [sort, setSort] = useState({ key: 'name', direction: 'asc' })
|
||||
const [page, setPage] = useState(1)
|
||||
const [selectedIds, setSelectedIds] = useState(new Set())
|
||||
const [activeAttention, setActiveAttention] = useState(null)
|
||||
const [inspectedId, setInspectedId] = useState(null)
|
||||
const [confirmRemove, setConfirmRemove] = useState(false)
|
||||
const [confirmStopModel, setConfirmStopModel] = useState(null)
|
||||
const [stoppingModelName, setStoppingModelName] = useState(null)
|
||||
const [bulkRunning, setBulkRunning] = useState(false)
|
||||
const bulkRunningRef = useRef(false)
|
||||
const [showTips, setShowTips] = useState(false)
|
||||
const [activeTab, setActiveTab] = useState('all') // 'all' | 'backend' | 'agent'
|
||||
const [emptyNodeType, setEmptyNodeType] = useState('backend')
|
||||
const [workbenchView, setWorkbenchView] = useState('nodes')
|
||||
const [modelRows, setModelRows] = useState([])
|
||||
const [modelLoadState, setModelLoadState] = useState('idle')
|
||||
const [modelError, setModelError] = useState('')
|
||||
const [modelQuery, setModelQuery] = useState('')
|
||||
const [modelSort, setModelSort] = useState({ key: 'model_name', direction: 'asc' })
|
||||
const [modelPage, setModelPage] = useState(1)
|
||||
const [inspectedModelName, setInspectedModelName] = useState(null)
|
||||
const [modelDrillNodeId, setModelDrillNodeId] = useState(null)
|
||||
const [returnFocusNodeId, setReturnFocusNodeId] = useState(null)
|
||||
const modelRequestStarted = useRef(false)
|
||||
const modelStopRunningRef = useRef(false)
|
||||
const modelStopInvokerRef = useRef(null)
|
||||
const nodesTabRef = useRef(null)
|
||||
const modelsTabRef = useRef(null)
|
||||
const nodeInvokerRef = useRef(null)
|
||||
const modelInvokerRef = useRef(null)
|
||||
|
||||
const fetchNodes = useCallback(async () => {
|
||||
try {
|
||||
const data = await nodesApi.list()
|
||||
setNodesList(Array.isArray(data) ? data : [])
|
||||
const nextNodes = Array.isArray(data) ? data : []
|
||||
setNodes(nextNodes)
|
||||
setSelectedIds(current => {
|
||||
const existing = new Set(nextNodes.map(node => node.id))
|
||||
return new Set([...current].filter(id => existing.has(id)))
|
||||
})
|
||||
setEnabled(true)
|
||||
} catch (err) {
|
||||
if (err.message?.includes('503') || err.message?.includes('Service Unavailable')) {
|
||||
setEnabled(false)
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.message?.includes('503') || error.message?.includes('Service Unavailable')) setEnabled(false)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Roster model fetch: drives the inline model chips on each backend panel
|
||||
// without an expand click. Grouped by node below.
|
||||
const fetchAllModels = useCallback(async () => {
|
||||
try {
|
||||
const d = await nodesApi.allModels()
|
||||
setAllModels(Array.isArray(d) ? d : [])
|
||||
} catch {
|
||||
setAllModels([])
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
fetchNodes()
|
||||
fetchAllModels()
|
||||
const interval = setInterval(() => {
|
||||
fetchNodes()
|
||||
fetchAllModels()
|
||||
}, 5000)
|
||||
const interval = setInterval(fetchNodes, 5000)
|
||||
return () => clearInterval(interval)
|
||||
}, [fetchNodes, fetchAllModels])
|
||||
}, [fetchNodes])
|
||||
|
||||
const modelsByNode = useMemo(() => {
|
||||
const m = {}
|
||||
for (const x of allModels) (m[x.node_id] ||= []).push(x)
|
||||
return m
|
||||
}, [allModels])
|
||||
const summary = useMemo(() => summarizeFleet(nodes), [nodes])
|
||||
const labelKeys = useMemo(() => [...new Set(nodes.flatMap(node => Object.keys(node.labels || {})))].sort(), [nodes])
|
||||
const attentionIds = useMemo(() => {
|
||||
if (!activeAttention) return null
|
||||
const values = activeAttention === 'all' ? Object.values(summary.attention).flat() : summary.attention[activeAttention]
|
||||
return new Set(values)
|
||||
}, [activeAttention, summary])
|
||||
const filtered = useMemo(() => {
|
||||
const result = filterNodes(nodes, { query, statuses: status ? [status] : [], types: type ? [type] : [] })
|
||||
return attentionIds ? result.filter(node => attentionIds.has(node.id)) : result
|
||||
}, [nodes, query, status, type, attentionIds])
|
||||
const ordered = useMemo(() => sortNodes(filtered, sort), [filtered, sort])
|
||||
const pagination = useMemo(() => paginateNodes(ordered, page), [ordered, page])
|
||||
const inspectedNode = nodes.find(node => node.id === inspectedId) || null
|
||||
const drilledNode = nodes.find(node => node.id === modelDrillNodeId) || null
|
||||
const groupedModels = useMemo(() => groupModels(modelRows), [modelRows])
|
||||
const filteredModels = useMemo(() => filterModels(groupedModels, modelQuery), [groupedModels, modelQuery])
|
||||
const orderedModels = useMemo(() => sortModels(filteredModels, modelSort), [filteredModels, modelSort])
|
||||
const modelPagination = useMemo(() => paginateModels(orderedModels, modelPage), [orderedModels, modelPage])
|
||||
const inspectedModel = groupedModels.find(model => model.model_name === inspectedModelName) || null
|
||||
|
||||
const handleDrain = async (nodeId) => {
|
||||
useEffect(() => { if (pagination.page !== page) setPage(pagination.page) }, [page, pagination.page])
|
||||
useEffect(() => { setPage(1) }, [query, status, type, activeAttention, groupBy])
|
||||
useEffect(() => { if (modelPagination.page !== modelPage) setModelPage(modelPagination.page) }, [modelPage, modelPagination.page])
|
||||
useEffect(() => { setModelPage(1) }, [modelQuery])
|
||||
|
||||
const refreshModels = useCallback(async () => {
|
||||
const data = await nodesApi.allModels()
|
||||
setModelRows(Array.isArray(data) ? data : [])
|
||||
setModelLoadState('loaded')
|
||||
setModelError('')
|
||||
}, [])
|
||||
|
||||
const loadModels = useCallback(async () => {
|
||||
if (modelRequestStarted.current) return
|
||||
modelRequestStarted.current = true
|
||||
setModelLoadState('loading')
|
||||
setModelError('')
|
||||
try {
|
||||
await nodesApi.drain(nodeId)
|
||||
addToast('Node set to draining', 'success')
|
||||
fetchNodes()
|
||||
} catch (err) {
|
||||
addToast(`Failed to drain node: ${err.message}`, 'error')
|
||||
await refreshModels()
|
||||
} catch (error) {
|
||||
setModelError(error.message || 'Unable to load running models')
|
||||
setModelLoadState('error')
|
||||
}
|
||||
}, [refreshModels])
|
||||
|
||||
const stopModel = () => {
|
||||
const model = confirmStopModel
|
||||
if (!model || modelStopRunningRef.current) return
|
||||
modelStopRunningRef.current = true
|
||||
setStoppingModelName(model.model_name)
|
||||
|
||||
void (async () => {
|
||||
let stopError = null
|
||||
try {
|
||||
await backendControlApi.shutdown({ model: model.model_name })
|
||||
} catch (error) {
|
||||
stopError = error
|
||||
}
|
||||
|
||||
try {
|
||||
await refreshModels()
|
||||
} catch (error) {
|
||||
setModelError(error.message || 'Unable to refresh running models')
|
||||
setModelLoadState('error')
|
||||
}
|
||||
|
||||
if (stopError) {
|
||||
addToast(`Could not stop ${model.model_name}: ${stopError.message || stopError}. Some replicas may already have stopped.`, 'warning')
|
||||
} else {
|
||||
addToast(`Stopped ${model.model_name}: ${model.replica_count} replica${model.replica_count === 1 ? '' : 's'} across ${model.node_count} node${model.node_count === 1 ? '' : 's'}.`, 'success')
|
||||
}
|
||||
setConfirmStopModel(null)
|
||||
setStoppingModelName(null)
|
||||
modelStopRunningRef.current = false
|
||||
})()
|
||||
}
|
||||
|
||||
const promptStopModel = (model, invoker) => {
|
||||
modelStopInvokerRef.current = invoker
|
||||
setConfirmStopModel(model)
|
||||
}
|
||||
|
||||
const cancelStopModel = () => {
|
||||
setConfirmStopModel(null)
|
||||
restoreFocus(modelStopInvokerRef)
|
||||
}
|
||||
|
||||
const activateWorkbench = view => {
|
||||
setWorkbenchView(view)
|
||||
setInspectedId(null)
|
||||
setInspectedModelName(null)
|
||||
setModelDrillNodeId(null)
|
||||
if (view === 'models') void loadModels()
|
||||
}
|
||||
|
||||
const handleTabKeyDown = event => {
|
||||
if (event.key !== 'ArrowLeft' && event.key !== 'ArrowRight') return
|
||||
event.preventDefault()
|
||||
const nextView = workbenchView === 'nodes' ? 'models' : 'nodes'
|
||||
activateWorkbench(nextView)
|
||||
;(nextView === 'nodes' ? nodesTabRef : modelsTabRef).current?.focus()
|
||||
}
|
||||
|
||||
const restoreFocus = ref => requestAnimationFrame(() => ref.current?.focus())
|
||||
|
||||
const openNodeInspector = (node, invoker) => {
|
||||
nodeInvokerRef.current = invoker
|
||||
setInspectedId(node.id)
|
||||
}
|
||||
|
||||
const closeNodeInspector = () => {
|
||||
setInspectedId(null)
|
||||
restoreFocus(nodeInvokerRef)
|
||||
}
|
||||
|
||||
const openModelInspector = (model, invoker) => {
|
||||
modelInvokerRef.current = invoker
|
||||
setInspectedModelName(model.model_name)
|
||||
setModelDrillNodeId(null)
|
||||
setReturnFocusNodeId(null)
|
||||
}
|
||||
|
||||
const closeModelDrilldown = () => {
|
||||
setInspectedModelName(null)
|
||||
setModelDrillNodeId(null)
|
||||
setReturnFocusNodeId(null)
|
||||
restoreFocus(modelInvokerRef)
|
||||
}
|
||||
|
||||
const openModelNode = node => {
|
||||
setReturnFocusNodeId(null)
|
||||
setModelDrillNodeId(node.id)
|
||||
}
|
||||
|
||||
const returnToModel = () => {
|
||||
setReturnFocusNodeId(modelDrillNodeId)
|
||||
setModelDrillNodeId(null)
|
||||
}
|
||||
|
||||
const actOnNode = async (action, nodeId, successMessage) => {
|
||||
try {
|
||||
await nodesApi[action](nodeId)
|
||||
addToast(successMessage, 'success')
|
||||
await fetchNodes()
|
||||
} catch (error) {
|
||||
addToast(error.message, 'error')
|
||||
}
|
||||
}
|
||||
|
||||
const handleResume = async (nodeId) => {
|
||||
try {
|
||||
await nodesApi.resume(nodeId)
|
||||
addToast('Node resumed', 'success')
|
||||
fetchNodes()
|
||||
} catch (err) {
|
||||
addToast(`Failed to resume node: ${err.message}`, 'error')
|
||||
}
|
||||
const runBulk = (action) => {
|
||||
if (bulkRunningRef.current) return
|
||||
bulkRunningRef.current = true
|
||||
setBulkRunning(true)
|
||||
|
||||
const ids = [...selectedIds]
|
||||
const requiredStatus = action === 'drain' ? 'healthy' : action === 'resume' ? 'draining' : null
|
||||
const statusById = new Map(nodes.map(node => [node.id, node.status]))
|
||||
const eligibleIds = requiredStatus ? ids.filter(id => statusById.get(id) === requiredStatus) : ids
|
||||
const skipped = ids.length - eligibleIds.length
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const results = await runBounded(eligibleIds, 8, id => nodesApi[action](id))
|
||||
const succeeded = results.filter(result => result.status === 'fulfilled').length
|
||||
const failed = results.length - succeeded
|
||||
const label = action === 'delete' ? 'Remove' : action[0].toUpperCase() + action.slice(1)
|
||||
addToast(`${label} complete: ${succeeded} succeeded, ${failed} failed, ${skipped} skipped`, failed || skipped ? 'warning' : 'success')
|
||||
await fetchNodes()
|
||||
} finally {
|
||||
setConfirmRemove(false)
|
||||
bulkRunningRef.current = false
|
||||
setBulkRunning(false)
|
||||
}
|
||||
})()
|
||||
}
|
||||
|
||||
const handleApprove = async (nodeId) => {
|
||||
try {
|
||||
await nodesApi.approve(nodeId)
|
||||
addToast('Node approved', 'success')
|
||||
fetchNodes()
|
||||
} catch (err) {
|
||||
addToast(`Failed to approve node: ${err.message}`, 'error')
|
||||
}
|
||||
}
|
||||
|
||||
const handleDelete = async (nodeId) => {
|
||||
try {
|
||||
await nodesApi.delete(nodeId)
|
||||
addToast('Node removed', 'success')
|
||||
setConfirmDelete(null)
|
||||
fetchNodes()
|
||||
} catch (err) {
|
||||
addToast(`Failed to remove node: ${err.message}`, 'error')
|
||||
setConfirmDelete(null)
|
||||
}
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="page page--wide loading-center">
|
||||
<LoadingSpinner size="lg" />
|
||||
if (loading) return <div className="page page--wide loading-center"><LoadingSpinner size="lg" /></div>
|
||||
if (!enabled) return <DisabledState addToast={addToast} />
|
||||
if (nodes.length === 0) return (
|
||||
<div className="page page--wide">
|
||||
<PageHeader title={t('nodes.title')} supporting={t('nodes.subtitle')} />
|
||||
<div role="radiogroup" aria-label="Worker type" className="segmented node-filter">
|
||||
{[['backend', 'Backend'], ['agent', 'Agent']].map(([value, label]) => <button key={value} type="button" role="radio" aria-checked={emptyNodeType === value} className={`segmented__item${emptyNodeType === value ? ' is-active' : ''}`} onClick={() => setEmptyNodeType(value)}>{label}</button>)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Disabled state
|
||||
if (!enabled) {
|
||||
return (
|
||||
<div className="page page--wide">
|
||||
<div className="p2p-hero">
|
||||
<i className="fas fa-network-wired text-primary" />
|
||||
<h1 className="text-xl fw-semibold mb-sm">
|
||||
Distributed Mode Not Enabled
|
||||
</h1>
|
||||
<p className="text-secondary" style={{ maxWidth: 600, margin: '0 auto var(--spacing-xl)' }}>
|
||||
Enable distributed mode to manage backend nodes across multiple machines. Nodes self-register and are monitored for health, enabling horizontal scaling of model inference.
|
||||
</p>
|
||||
|
||||
<div className="nodes-features mb-xl">
|
||||
<div className="card text-center pad-md">
|
||||
<div className="icon-chip icon-chip--centred tone-primary">
|
||||
<i className="fas fa-server" />
|
||||
</div>
|
||||
<h3 className="text-base fw-semibold mb-xs">Horizontal Scaling</h3>
|
||||
<p className="text-sub">Add backend nodes to scale inference capacity</p>
|
||||
</div>
|
||||
<div className="card text-center pad-md">
|
||||
<div className="icon-chip icon-chip--centred tone-accent">
|
||||
<i className="fas fa-route" />
|
||||
</div>
|
||||
<h3 className="text-base fw-semibold mb-xs">Smart Routing</h3>
|
||||
<p className="text-sub">Route requests to the best available node</p>
|
||||
</div>
|
||||
<div className="card text-center pad-md">
|
||||
<div className="icon-chip icon-chip--centred tone-success">
|
||||
<i className="fas fa-heart-pulse" />
|
||||
</div>
|
||||
<h3 className="text-base fw-semibold mb-xs">Health Monitoring</h3>
|
||||
<p className="text-sub">Automatic heartbeat checks and failover</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="card p2p-enable pad-lg">
|
||||
<h3 className="panel-title">
|
||||
<i className="fas fa-rocket text-accent" />
|
||||
How to Enable Distributed Mode
|
||||
</h3>
|
||||
<div className="stack">
|
||||
<div className="hstack">
|
||||
<StepNumber n={1} bg="var(--color-accent-light)" color="var(--color-accent)" />
|
||||
<div className="flex-1">
|
||||
<p className="fw-medium mb-xs">Start LocalAI with distributed mode</p>
|
||||
<CommandBlock
|
||||
command={`local-ai run --distributed \\\n --distributed-db "postgres://user:pass@host/db" \\\n --distributed-nats "nats://host:4222"`}
|
||||
addToast={addToast}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="hstack">
|
||||
<StepNumber n={2} bg="var(--color-accent-light)" color="var(--color-accent)" />
|
||||
<div className="flex-1">
|
||||
<p className="fw-medium mb-xs">Register backend nodes</p>
|
||||
<CommandBlock
|
||||
command={`local-ai worker \\\n --register-to "http://localai-host:8080" \\\n --nats-url "nats://nats:4222" \\\n --node-name "gpu-node-1"`}
|
||||
addToast={addToast}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="hstack">
|
||||
<StepNumber n={3} bg="var(--color-accent-light)" color="var(--color-accent)" />
|
||||
<div className="flex-1">
|
||||
<p className="fw-medium">Manage nodes from this dashboard</p>
|
||||
<p className="text-sub mt-xs">
|
||||
Once enabled, refresh this page to see registered nodes and their health status.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-note mt-md">
|
||||
For full setup instructions, architecture details, and Kubernetes deployment, see the{' '}
|
||||
<a href="https://localai.io/features/distributed-mode/" target="_blank" rel="noopener noreferrer"
|
||||
className="text-primary">Distributed Mode documentation <i className="fas fa-external-link-alt text-xs" /></a>.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Split nodes by type
|
||||
const backendNodes = nodesList.filter(n => !n.node_type || n.node_type === 'backend')
|
||||
const agentNodes = nodesList.filter(n => n.node_type === 'agent')
|
||||
const filteredNodes = activeTab === 'all' ? nodesList
|
||||
: activeTab === 'agent' ? agentNodes : backendNodes
|
||||
<WorkerHintCard addToast={addToast} nodeType={emptyNodeType} />
|
||||
</div>
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="page page--wide">
|
||||
<PageHeader
|
||||
title={
|
||||
<>
|
||||
<i className="fas fa-network-wired icon-before" />
|
||||
{t('nodes.title')}
|
||||
</>
|
||||
}
|
||||
supporting={t('nodes.subtitle')}
|
||||
/>
|
||||
<div className={`page page--wide nodes-fleet-page${inspectedNode || inspectedModel || drilledNode ? ' nodes-fleet-page--inspecting' : ''}`}>
|
||||
<PageHeader className="nodes-fleet-page__header" eyebrow={null} title={t('nodes.title')} supporting={t('nodes.subtitle')} actions={<button type="button" className="btn btn-secondary btn-sm" onClick={() => setShowTips(value => !value)}>{showTips ? 'Hide setup' : 'Register worker'}</button>} />
|
||||
{showTips && <WorkerHintCard addToast={addToast} hasWorkers />}
|
||||
<ClusterOverview summary={summary} activeAttention={activeAttention} onAttentionSelect={setActiveAttention} />
|
||||
|
||||
<ClusterPulse nodes={nodesList} />
|
||||
<AttentionCallout nodes={nodesList} onApprove={handleApprove} />
|
||||
|
||||
{/* Node-type filter */}
|
||||
<div role="radiogroup" aria-label="Node type" className="segmented node-filter">
|
||||
{[['all', 'All'], ['backend', 'Backend'], ['agent', 'Agent']].map(([key, label]) => (
|
||||
<button key={key} type="button" role="radio" aria-checked={activeTab === key}
|
||||
className={`segmented__item${activeTab === key ? ' is-active' : ''}`}
|
||||
onClick={() => setActiveTab(key)}>{label}</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Worker tips */}
|
||||
{!loading && filteredNodes.length === 0 ? (
|
||||
<WorkerHintCard addToast={addToast} activeTab={activeTab} hasWorkers={false} />
|
||||
) : (
|
||||
<>
|
||||
<button
|
||||
onClick={() => setShowTips(t => !t)}
|
||||
className="nodes-add-worker"
|
||||
aria-expanded={showTips}
|
||||
>
|
||||
<i className={`fas ${showTips ? 'fa-chevron-down' : 'fa-plus'}`} />
|
||||
{showTips ? 'Hide instructions' : 'Register a new worker'}
|
||||
</button>
|
||||
{showTips && <WorkerHintCard addToast={addToast} activeTab={activeTab} hasWorkers />}
|
||||
</>
|
||||
)}
|
||||
|
||||
{filteredNodes.length > 0 && (
|
||||
<div className="node-roster">
|
||||
{filteredNodes.map(node => (
|
||||
<NodePanel key={node.id} node={node} models={modelsByNode[node.id] || []}
|
||||
onApprove={handleApprove} onDrain={handleDrain} onResume={handleResume}
|
||||
onRemove={(n) => setConfirmDelete(n)} />
|
||||
))}
|
||||
<section className="fleet-workbench" aria-label="Fleet workbench">
|
||||
<div className="fleet-workbench__tabs" role="tablist" aria-label="Fleet views">
|
||||
<button ref={nodesTabRef} id="fleet-nodes-tab" type="button" role="tab" aria-selected={workbenchView === 'nodes'} aria-controls="fleet-nodes-panel"
|
||||
tabIndex={workbenchView === 'nodes' ? 0 : -1} className={workbenchView === 'nodes' ? 'is-active' : ''} onKeyDown={handleTabKeyDown} onClick={() => activateWorkbench('nodes')}>Nodes <span>{nodes.length}</span></button>
|
||||
<button ref={modelsTabRef} id="fleet-models-tab" type="button" role="tab" aria-selected={workbenchView === 'models'} aria-controls="fleet-models-panel"
|
||||
tabIndex={workbenchView === 'models' ? 0 : -1} className={workbenchView === 'models' ? 'is-active' : ''} onKeyDown={handleTabKeyDown} onClick={() => activateWorkbench('models')}>Running models <span>{modelLoadState === 'loaded' ? groupedModels.length : '—'}</span></button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ConfirmDialog
|
||||
open={!!confirmDelete}
|
||||
title="Remove Node"
|
||||
message={confirmDelete ? `Are you sure you want to remove node "${confirmDelete.name}"? This will deregister it from the cluster.` : ''}
|
||||
confirmLabel="Remove"
|
||||
danger
|
||||
onConfirm={() => confirmDelete && handleDelete(confirmDelete.id)}
|
||||
onCancel={() => setConfirmDelete(null)}
|
||||
/>
|
||||
|
||||
<div className="fleet-workbench__layout">
|
||||
<div id="fleet-nodes-panel" className="fleet-workbench__fleet" role="tabpanel" aria-labelledby="fleet-nodes-tab" hidden={workbenchView !== 'nodes'}>
|
||||
<div className="fleet-toolbar">
|
||||
<input className="input fleet-toolbar__search" type="search" aria-label="Search nodes" placeholder="Search name, address, label…" value={query} onChange={event => setQuery(event.target.value)} />
|
||||
<FleetSelect label="Filter status" value={status} onChange={event => setStatus(event.target.value)}><option value="">All statuses</option>{['healthy', 'draining', 'pending', 'unhealthy', 'offline'].map(value => <option key={value} value={value}>{value}</option>)}</FleetSelect>
|
||||
<FleetSelect label="Filter type" value={type} onChange={event => setType(event.target.value)}><option value="">All types</option><option value="backend">backend</option><option value="agent">agent</option></FleetSelect>
|
||||
<FleetSelect label="Group nodes" value={groupBy} onChange={event => setGroupBy(event.target.value)}><option value="none">No grouping</option><option value="node_type">Group by type</option>{labelKeys.map(key => <option key={key} value={`label:${key}`}>Label: {key}</option>)}</FleetSelect>
|
||||
</div>
|
||||
{selectedIds.size > 0 && <div className="fleet-bulkbar">
|
||||
<strong>{selectedIds.size} selected</strong>
|
||||
<button type="button" className="btn btn-secondary btn-sm" disabled={bulkRunning} onClick={() => runBulk('drain')}>Drain selected</button>
|
||||
<button type="button" className="btn btn-secondary btn-sm" disabled={bulkRunning} onClick={() => runBulk('resume')}>Resume selected</button>
|
||||
<button type="button" className="btn btn-danger btn-sm" disabled={bulkRunning} onClick={() => setConfirmRemove(true)}>Remove selected</button>
|
||||
<button type="button" className="fleet-bulkbar__clear" disabled={bulkRunning} onClick={() => setSelectedIds(new Set())}>Clear selection</button>
|
||||
<span className="fleet-bulkbar__count" aria-live="polite">{ordered.length} nodes in view</span>
|
||||
</div>}
|
||||
<NodeFleetTable nodes={pagination.items} selectedIds={selectedIds} onSelectionChange={setSelectedIds} onInspect={openNodeInspector} sort={sort} onSortChange={setSort} groupBy={groupBy}
|
||||
onApprove={id => actOnNode('approve', id, 'Node approved')} />
|
||||
<div className="fleet-pagination"><span>Page {pagination.page} of {pagination.totalPages}</span><button type="button" className="btn btn-secondary btn-sm" aria-label="Previous page" disabled={pagination.page === 1} onClick={() => setPage(value => value - 1)}>Previous</button><button type="button" className="btn btn-secondary btn-sm" aria-label="Next page" disabled={pagination.page === pagination.totalPages} onClick={() => setPage(value => value + 1)}>Next</button></div>
|
||||
</div>
|
||||
<div id="fleet-models-panel" className="fleet-workbench__fleet model-workbench" role="tabpanel" aria-labelledby="fleet-models-tab" hidden={workbenchView !== 'models'}>
|
||||
<div className="model-workbench__scope"><div><strong>Running models</strong><span>Current loaded replicas on healthy nodes</span></div>{modelLoadState === 'loaded' && <span aria-live="polite">{orderedModels.length} model{orderedModels.length === 1 ? '' : 's'} in view</span>}</div>
|
||||
{modelLoadState === 'loading' && <div className="model-workbench__state" role="status"><LoadingSpinner size="sm" /><strong>Loading running models…</strong><span>Reading the controller's current replica inventory.</span></div>}
|
||||
{modelLoadState === 'error' && <div className="model-workbench__state model-workbench__state--error" role="alert"><i className="fas fa-triangle-exclamation" aria-hidden="true" /><strong>Unable to load running models</strong><span>{modelError}</span><button type="button" className="btn btn-secondary btn-sm" aria-label="Retry loading running models" onClick={() => { modelRequestStarted.current = false; void loadModels() }}>Retry</button></div>}
|
||||
{modelLoadState === 'loaded' && groupedModels.length === 0 && <div className="model-workbench__state"><i className="fas fa-layer-group" aria-hidden="true" /><strong>No running models</strong><span>Loaded replicas on healthy nodes will appear here.</span></div>}
|
||||
{modelLoadState === 'loaded' && groupedModels.length > 0 && <>
|
||||
<div className="model-toolbar"><input className="input fleet-toolbar__search" type="search" aria-label="Search running models" placeholder="Search model or backend…" value={modelQuery} onChange={event => setModelQuery(event.target.value)} /></div>
|
||||
<ModelFleetTable models={modelPagination.items} selectedName={inspectedModelName} inspectorOpen={!!inspectedModel && !drilledNode} onInspect={openModelInspector}
|
||||
onStop={promptStopModel} stoppingName={stoppingModelName} sort={modelSort} onSortChange={setModelSort} />
|
||||
<div className="fleet-pagination"><span>Page {modelPagination.page} of {modelPagination.totalPages}</span><button type="button" className="btn btn-secondary btn-sm" aria-label="Previous model page" disabled={modelPagination.page === 1} onClick={() => setModelPage(value => value - 1)}>Previous</button><button type="button" className="btn btn-secondary btn-sm" aria-label="Next model page" disabled={modelPagination.page === modelPagination.totalPages} onClick={() => setModelPage(value => value + 1)}>Next</button></div>
|
||||
</>}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
{workbenchView === 'nodes' && <NodeInspector node={inspectedNode} open={!!inspectedNode} onClose={closeNodeInspector}
|
||||
onApprove={id => actOnNode('approve', id, 'Node approved')}
|
||||
onDrain={id => actOnNode('drain', id, 'Node set to draining')} onResume={id => actOnNode('resume', id, 'Node resumed')} />
|
||||
}
|
||||
{workbenchView === 'models' && !drilledNode && <ModelInspector model={inspectedModel} nodes={nodes} open={!!inspectedModel} onClose={closeModelDrilldown} onOpenNode={openModelNode} focusNodeId={returnFocusNodeId} />}
|
||||
{workbenchView === 'models' && drilledNode && <NodeInspector node={drilledNode} open onClose={closeModelDrilldown}
|
||||
onBack={returnToModel} backLabel={`Back to ${inspectedModel?.model_name || 'model'}`}
|
||||
onApprove={id => actOnNode('approve', id, 'Node approved')}
|
||||
onDrain={id => actOnNode('drain', id, 'Node set to draining')} onResume={id => actOnNode('resume', id, 'Node resumed')} />}
|
||||
<ConfirmDialog open={confirmRemove} title="Remove selected nodes" message={`Remove ${selectedIds.size} selected nodes from the cluster?`} confirmLabel="Remove nodes" pendingLabel="Removing…" pending={bulkRunning} danger onConfirm={() => runBulk('delete')} onCancel={() => setConfirmRemove(false)} />
|
||||
<ConfirmDialog open={!!confirmStopModel} title={confirmStopModel ? `Stop ${confirmStopModel.model_name}?` : 'Stop model?'}
|
||||
message={confirmStopModel ? `${confirmStopModel.model_name} has ${confirmStopModel.replica_count} loaded replica${confirmStopModel.replica_count === 1 ? '' : 's'} across ${confirmStopModel.node_count} unique node${confirmStopModel.node_count === 1 ? '' : 's'}. This will stop all loaded placements on those nodes.` : ''}
|
||||
confirmLabel="Stop model" pendingLabel="Stopping…" pending={!!stoppingModelName} danger onConfirm={stopModel} onCancel={cancelStopModel} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
const HEALTH_STATUSES = ['healthy', 'unhealthy', 'offline', 'pending', 'draining']
|
||||
|
||||
function finiteNumber(value) {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null
|
||||
}
|
||||
|
||||
function clamp(value, minimum, maximum) {
|
||||
return Math.min(maximum, Math.max(minimum, value))
|
||||
}
|
||||
|
||||
export function capacityReading(totalValue, availableValue) {
|
||||
const total = finiteNumber(totalValue)
|
||||
const reportedAvailable = finiteNumber(availableValue)
|
||||
if (total === null || total <= 0 || reportedAvailable === null) return null
|
||||
|
||||
const available = clamp(reportedAvailable, 0, total)
|
||||
const used = total - available
|
||||
return { total, used, available, usagePercent: (used / total) * 100 }
|
||||
}
|
||||
|
||||
export function nodeLifecycleAction(status) {
|
||||
if (status === 'healthy') return 'drain'
|
||||
if (status === 'draining') return 'resume'
|
||||
if (status === 'pending') return 'approve'
|
||||
return null
|
||||
}
|
||||
|
||||
function capacitySummary(nodes, totalField, availableField) {
|
||||
let total = 0
|
||||
let available = 0
|
||||
let reportingCount = 0
|
||||
|
||||
for (const node of nodes) {
|
||||
const reading = capacityReading(node?.[totalField], node?.[availableField])
|
||||
if (!reading) continue
|
||||
|
||||
total += reading.total
|
||||
available += reading.available
|
||||
reportingCount += 1
|
||||
}
|
||||
|
||||
const used = total - available
|
||||
return {
|
||||
total,
|
||||
used,
|
||||
available,
|
||||
usagePercent: total > 0 ? (used / total) * 100 : 0,
|
||||
reportingCount,
|
||||
unknownCount: nodes.length - reportingCount,
|
||||
}
|
||||
}
|
||||
|
||||
function cpuSummary(nodes) {
|
||||
let totalLogicalCores = 0
|
||||
let busyCoreEquivalents = 0
|
||||
let load1 = 0
|
||||
let reportingCount = 0
|
||||
|
||||
for (const node of nodes) {
|
||||
const logicalCores = finiteNumber(node?.cpu_logical_cores)
|
||||
const usage = finiteNumber(node?.cpu_usage_percent)
|
||||
const reportedLoad = finiteNumber(node?.cpu_load_1)
|
||||
if (logicalCores === null || logicalCores <= 0 || usage === null || reportedLoad === null) continue
|
||||
|
||||
const usagePercent = clamp(usage, 0, 100)
|
||||
totalLogicalCores += logicalCores
|
||||
busyCoreEquivalents += logicalCores * usagePercent / 100
|
||||
load1 += Math.max(0, reportedLoad)
|
||||
reportingCount += 1
|
||||
}
|
||||
|
||||
return {
|
||||
totalLogicalCores,
|
||||
busyCoreEquivalents,
|
||||
idleCoreEquivalents: totalLogicalCores - busyCoreEquivalents,
|
||||
usagePercent: totalLogicalCores > 0 ? (busyCoreEquivalents / totalLogicalCores) * 100 : 0,
|
||||
load1,
|
||||
reportingCount,
|
||||
unknownCount: nodes.length - reportingCount,
|
||||
}
|
||||
}
|
||||
|
||||
function hasLowCapacity(node, totalField, availableField) {
|
||||
const reading = capacityReading(node?.[totalField], node?.[availableField])
|
||||
return reading !== null && reading.available / reading.total <= 0.1
|
||||
}
|
||||
|
||||
export function summarizeFleet(input) {
|
||||
const nodes = Array.isArray(input) ? input : []
|
||||
const health = {
|
||||
total: nodes.length,
|
||||
healthy: 0,
|
||||
draining: 0,
|
||||
pending: 0,
|
||||
offline: 0,
|
||||
unhealthy: 0,
|
||||
other: 0,
|
||||
}
|
||||
|
||||
const attention = {
|
||||
pending: [],
|
||||
offlineOrUnhealthy: [],
|
||||
lowVRAM: [],
|
||||
lowRAM: [],
|
||||
lowDisk: [],
|
||||
}
|
||||
|
||||
nodes.forEach(node => {
|
||||
const status = typeof node?.status === 'string' ? node.status.toLowerCase() : ''
|
||||
if (HEALTH_STATUSES.includes(status)) health[status] += 1
|
||||
else health.other += 1
|
||||
|
||||
if (status === 'pending') attention.pending.push(node?.id)
|
||||
if (status === 'offline' || status === 'unhealthy') attention.offlineOrUnhealthy.push(node?.id)
|
||||
if (hasLowCapacity(node, 'total_vram', 'available_vram')) attention.lowVRAM.push(node?.id)
|
||||
if (hasLowCapacity(node, 'total_ram', 'available_ram')) attention.lowRAM.push(node?.id)
|
||||
if (hasLowCapacity(node, 'total_disk', 'available_disk')) attention.lowDisk.push(node?.id)
|
||||
})
|
||||
|
||||
const attentionIds = new Set(Object.values(attention).flat())
|
||||
|
||||
return {
|
||||
health,
|
||||
attentionNodeCount: attentionIds.size,
|
||||
attention,
|
||||
vram: capacitySummary(nodes, 'total_vram', 'available_vram'),
|
||||
ram: capacitySummary(nodes, 'total_ram', 'available_ram'),
|
||||
cpu: cpuSummary(nodes),
|
||||
disk: capacitySummary(nodes, 'total_disk', 'available_disk'),
|
||||
}
|
||||
}
|
||||
|
||||
function normalizedSet(values) {
|
||||
if (!values) return new Set()
|
||||
const iterable = values instanceof Set || Array.isArray(values) ? values : [values]
|
||||
return new Set([...iterable].map(value => String(value).toLowerCase()))
|
||||
}
|
||||
|
||||
function searchableValues(node) {
|
||||
const labels = node?.labels && typeof node.labels === 'object' && !Array.isArray(node.labels)
|
||||
? Object.entries(node.labels).flat()
|
||||
: []
|
||||
return [
|
||||
node?.name,
|
||||
node?.address,
|
||||
node?.node_type,
|
||||
node?.status,
|
||||
node?.model_count,
|
||||
node?.gpu_vendor,
|
||||
node?.capability,
|
||||
...labels,
|
||||
]
|
||||
}
|
||||
|
||||
export function filterNodes(input, filters = {}) {
|
||||
const nodes = Array.isArray(input) ? input : []
|
||||
const query = String(filters.query ?? '').trim().toLowerCase()
|
||||
const statuses = normalizedSet(filters.statuses)
|
||||
const types = normalizedSet(filters.types)
|
||||
|
||||
return nodes.filter(node => {
|
||||
const status = String(node?.status ?? '').toLowerCase()
|
||||
const type = String(node?.node_type ?? '').toLowerCase()
|
||||
if (statuses.size > 0 && !statuses.has(status)) return false
|
||||
if (types.size > 0 && !types.has(type)) return false
|
||||
if (!query) return true
|
||||
return searchableValues(node).some(value => String(value ?? '').toLowerCase().includes(query))
|
||||
})
|
||||
}
|
||||
|
||||
function compareValues(left, right) {
|
||||
const leftMissing = left == null || (typeof left === 'number' && !Number.isFinite(left))
|
||||
const rightMissing = right == null || (typeof right === 'number' && !Number.isFinite(right))
|
||||
if (leftMissing || rightMissing) return leftMissing === rightMissing ? 0 : leftMissing ? 1 : -1
|
||||
|
||||
const leftNumber = finiteNumber(left)
|
||||
const rightNumber = finiteNumber(right)
|
||||
if (leftNumber !== null && rightNumber !== null) return leftNumber - rightNumber
|
||||
return String(left).localeCompare(String(right), undefined, { numeric: true, sensitivity: 'base' })
|
||||
}
|
||||
|
||||
export function sortNodes(input, sort = {}) {
|
||||
const nodes = Array.isArray(input) ? input : []
|
||||
const key = sort.key || 'name'
|
||||
const direction = sort.direction === 'desc' ? -1 : 1
|
||||
|
||||
return nodes
|
||||
.map((node, index) => ({ node, index }))
|
||||
.sort((left, right) => {
|
||||
const primary = compareValues(left.node?.[key], right.node?.[key]) * direction
|
||||
if (primary !== 0) return primary
|
||||
const byName = compareValues(left.node?.name, right.node?.name)
|
||||
return byName || left.index - right.index
|
||||
})
|
||||
.map(entry => entry.node)
|
||||
}
|
||||
|
||||
function modelTimestamp(value) {
|
||||
if (typeof value !== 'string' || value.trim() === '') return null
|
||||
const timestamp = Date.parse(value)
|
||||
return Number.isFinite(timestamp) ? timestamp : null
|
||||
}
|
||||
|
||||
export function groupModels(input) {
|
||||
const rows = Array.isArray(input) ? input : []
|
||||
const groups = new Map()
|
||||
|
||||
for (const replica of rows) {
|
||||
const modelName = typeof replica?.model_name === 'string' ? replica.model_name.trim() : ''
|
||||
if (!modelName) continue
|
||||
if (!groups.has(modelName)) {
|
||||
groups.set(modelName, {
|
||||
model_name: modelName,
|
||||
replicas: [],
|
||||
replica_count: 0,
|
||||
node_count: 0,
|
||||
in_flight: 0,
|
||||
backend_types: [],
|
||||
last_used: null,
|
||||
})
|
||||
}
|
||||
|
||||
const group = groups.get(modelName)
|
||||
group.replicas.push(replica)
|
||||
group.replica_count += 1
|
||||
const inFlight = finiteNumber(replica.in_flight)
|
||||
group.in_flight += inFlight === null ? 0 : Math.max(0, Math.floor(inFlight))
|
||||
}
|
||||
|
||||
for (const group of groups.values()) {
|
||||
group.node_count = new Set(group.replicas.map(replica => String(replica?.node_id ?? '').trim()).filter(Boolean)).size
|
||||
group.backend_types = [...new Set(group.replicas.map(replica => String(replica?.backend_type ?? '').trim()).filter(Boolean))]
|
||||
.sort((left, right) => compareValues(left, right))
|
||||
const mostRecent = group.replicas.reduce((latest, replica) => {
|
||||
const timestamp = modelTimestamp(replica?.last_used)
|
||||
return timestamp !== null && (latest === null || timestamp > latest.timestamp)
|
||||
? { timestamp, value: replica.last_used }
|
||||
: latest
|
||||
}, null)
|
||||
group.last_used = mostRecent?.value ?? null
|
||||
}
|
||||
|
||||
return [...groups.values()]
|
||||
}
|
||||
|
||||
export function filterModels(input, query = '') {
|
||||
const models = Array.isArray(input) ? input : []
|
||||
const normalizedQuery = String(query ?? '').trim().toLowerCase()
|
||||
if (!normalizedQuery) return [...models]
|
||||
return models.filter(model => [model?.model_name, ...(Array.isArray(model?.backend_types) ? model.backend_types : [])]
|
||||
.some(value => String(value ?? '').toLowerCase().includes(normalizedQuery)))
|
||||
}
|
||||
|
||||
export function sortModels(input, sort = {}) {
|
||||
const models = Array.isArray(input) ? input : []
|
||||
const key = sort.key || 'model_name'
|
||||
const direction = sort.direction === 'desc' ? -1 : 1
|
||||
|
||||
return models
|
||||
.map((model, index) => ({ model, index }))
|
||||
.sort((left, right) => {
|
||||
let primary
|
||||
if (key === 'last_used') {
|
||||
const leftTime = modelTimestamp(left.model?.last_used)
|
||||
const rightTime = modelTimestamp(right.model?.last_used)
|
||||
if (leftTime === null || rightTime === null) primary = leftTime === rightTime ? 0 : leftTime === null ? 1 : -1
|
||||
else primary = (leftTime - rightTime) * direction
|
||||
} else {
|
||||
primary = compareValues(left.model?.[key], right.model?.[key]) * direction
|
||||
}
|
||||
if (primary !== 0) return primary
|
||||
const byName = compareValues(left.model?.model_name, right.model?.model_name)
|
||||
return byName || left.index - right.index
|
||||
})
|
||||
.map(entry => entry.model)
|
||||
}
|
||||
|
||||
export function paginateModels(input, requestedPage = 1) {
|
||||
return paginateNodes(input, requestedPage, 50)
|
||||
}
|
||||
|
||||
function groupDescriptor(node, groupBy) {
|
||||
if (groupBy === 'node_type') {
|
||||
const value = node?.node_type
|
||||
if (value == null || String(value).trim() === '') return { key: 'node-type:missing', label: 'Unlabelled', missing: true }
|
||||
return { key: `node-type:value:${JSON.stringify(String(value))}`, label: String(value), missing: false }
|
||||
}
|
||||
|
||||
if (typeof groupBy === 'string' && groupBy.startsWith('label:')) {
|
||||
const labelKey = groupBy.slice('label:'.length)
|
||||
const labels = node?.labels && typeof node.labels === 'object' && !Array.isArray(node.labels) ? node.labels : {}
|
||||
const value = Object.hasOwn(labels, labelKey) ? labels[labelKey] : undefined
|
||||
if (value == null || String(value).trim() === '') return { key: 'label:missing', label: 'Unlabelled', missing: true }
|
||||
return { key: `label:value:${JSON.stringify(String(value))}`, label: String(value), missing: false }
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export function groupNodes(input, groupBy = 'none') {
|
||||
const nodes = Array.isArray(input) ? input : []
|
||||
if (groupBy === 'none' || !groupBy) return [{ key: 'all', label: 'All nodes', nodes: [...nodes] }]
|
||||
|
||||
const groups = new Map()
|
||||
for (const node of nodes) {
|
||||
const descriptor = groupDescriptor(node, groupBy)
|
||||
if (!descriptor) return [{ key: 'all', label: 'All nodes', nodes: [...nodes] }]
|
||||
if (!groups.has(descriptor.key)) groups.set(descriptor.key, { ...descriptor, nodes: [] })
|
||||
groups.get(descriptor.key).nodes.push(node)
|
||||
}
|
||||
|
||||
return [...groups.values()]
|
||||
.sort((left, right) => {
|
||||
if (left.missing !== right.missing) return left.missing ? 1 : -1
|
||||
return compareValues(left.label, right.label)
|
||||
})
|
||||
.map(({ key, label, nodes: groupedNodes }) => ({ key, label, nodes: groupedNodes }))
|
||||
}
|
||||
|
||||
export function paginateNodes(input, requestedPage = 1, requestedPageSize = 50) {
|
||||
const nodes = Array.isArray(input) ? input : []
|
||||
const numericPageSize = Number.isFinite(requestedPageSize) ? Math.floor(requestedPageSize) : 50
|
||||
const pageSize = numericPageSize > 0 ? numericPageSize : 50
|
||||
const totalItems = nodes.length
|
||||
const totalPages = Math.max(1, Math.ceil(totalItems / pageSize))
|
||||
const numericPage = Number.isFinite(Number(requestedPage)) ? Math.floor(Number(requestedPage)) : 1
|
||||
const page = clamp(numericPage, 1, totalPages)
|
||||
const start = (page - 1) * pageSize
|
||||
|
||||
return {
|
||||
page,
|
||||
pageSize,
|
||||
totalItems,
|
||||
totalPages,
|
||||
items: nodes.slice(start, start + pageSize),
|
||||
}
|
||||
}
|
||||
|
||||
export async function runBounded(input, limit, operation) {
|
||||
const items = Array.isArray(input) ? input : []
|
||||
if (!Number.isInteger(limit) || limit <= 0) throw new RangeError('limit must be a positive integer')
|
||||
if (typeof operation !== 'function') throw new TypeError('operation must be a function')
|
||||
|
||||
const results = new Array(items.length)
|
||||
let nextIndex = 0
|
||||
|
||||
async function worker() {
|
||||
while (nextIndex < items.length) {
|
||||
const index = nextIndex
|
||||
nextIndex += 1
|
||||
try {
|
||||
results[index] = { status: 'fulfilled', value: await operation(items[index], index) }
|
||||
} catch (reason) {
|
||||
results[index] = { status: 'rejected', reason }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker())
|
||||
await Promise.all(workers)
|
||||
return results
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
|
||||
import {
|
||||
capacityReading,
|
||||
filterModels,
|
||||
filterNodes,
|
||||
groupModels,
|
||||
groupNodes,
|
||||
paginateModels,
|
||||
paginateNodes,
|
||||
runBounded,
|
||||
sortModels,
|
||||
sortNodes,
|
||||
summarizeFleet,
|
||||
nodeLifecycleAction,
|
||||
} from './nodeFleet.js'
|
||||
|
||||
const modelRows = [
|
||||
{ id: 'r1', node_id: 'n1', model_name: 'Llama 3.2', replica_index: 0, address: '10.0.0.1:50051', in_flight: 2, backend_type: 'llama-cpp', last_used: '2026-09-14T10:00:00Z' },
|
||||
{ id: 'r2', node_id: 'n1', model_name: 'Llama 3.2', replica_index: 1, address: '10.0.0.1:50052', in_flight: -4, backend_type: 'llama-cpp', last_used: 'invalid' },
|
||||
{ id: 'r3', node_id: 'n2', model_name: 'Llama 3.2', replica_index: 0, address: '10.0.0.2:50051', in_flight: 3.8, backend_type: ' vllm ', last_used: '2026-09-14T11:00:00Z' },
|
||||
{ id: 'r4', node_id: '', model_name: 'Whisper', replica_index: 0, address: '', in_flight: Number.NaN, backend_type: '', last_used: null },
|
||||
]
|
||||
|
||||
test('groups model replicas across nodes and normalizes defensive aggregate values', () => {
|
||||
const grouped = groupModels(modelRows)
|
||||
|
||||
assert.equal(grouped.length, 2)
|
||||
assert.deepEqual(grouped[0], {
|
||||
model_name: 'Llama 3.2',
|
||||
replicas: modelRows.slice(0, 3),
|
||||
replica_count: 3,
|
||||
node_count: 2,
|
||||
in_flight: 5,
|
||||
backend_types: ['llama-cpp', 'vllm'],
|
||||
last_used: '2026-09-14T11:00:00Z',
|
||||
})
|
||||
assert.deepEqual(grouped[1], {
|
||||
model_name: 'Whisper',
|
||||
replicas: [modelRows[3]],
|
||||
replica_count: 1,
|
||||
node_count: 0,
|
||||
in_flight: 0,
|
||||
backend_types: [],
|
||||
last_used: null,
|
||||
})
|
||||
})
|
||||
|
||||
test('ignores malformed model rows while preserving valid replicas and input order', () => {
|
||||
const input = [null, {}, { model_name: ' ' }, ...modelRows]
|
||||
const original = [...input]
|
||||
|
||||
assert.deepEqual(groupModels(input).flatMap(model => model.replicas), modelRows)
|
||||
assert.deepEqual(input, original)
|
||||
assert.deepEqual(groupModels(null), [])
|
||||
})
|
||||
|
||||
test('filters grouped models by name and backend type case-insensitively', () => {
|
||||
const grouped = groupModels(modelRows)
|
||||
|
||||
assert.deepEqual(filterModels(grouped, 'LLAMA').map(model => model.model_name), ['Llama 3.2'])
|
||||
assert.deepEqual(filterModels(grouped, 'VLLM').map(model => model.model_name), ['Llama 3.2'])
|
||||
assert.equal(filterModels(grouped, '').length, 2)
|
||||
})
|
||||
|
||||
test('sorts grouped models stably across every roster column without mutation', () => {
|
||||
const input = [
|
||||
{ model_name: 'Zulu', replica_count: 2, node_count: 1, in_flight: 4, last_used: null },
|
||||
{ model_name: 'Alpha', replica_count: 2, node_count: 2, in_flight: 1, last_used: '2026-09-14T09:00:00Z' },
|
||||
{ model_name: 'Beta', replica_count: 1, node_count: 3, in_flight: 1, last_used: '2026-09-14T10:00:00Z' },
|
||||
]
|
||||
const original = [...input]
|
||||
|
||||
assert.deepEqual(sortModels(input, { key: 'replica_count', direction: 'desc' }).map(model => model.model_name), ['Alpha', 'Zulu', 'Beta'])
|
||||
assert.deepEqual(sortModels(input, { key: 'node_count', direction: 'desc' }).map(model => model.model_name), ['Beta', 'Alpha', 'Zulu'])
|
||||
assert.deepEqual(sortModels(input, { key: 'in_flight', direction: 'asc' }).map(model => model.model_name), ['Alpha', 'Beta', 'Zulu'])
|
||||
assert.deepEqual(sortModels(input, { key: 'last_used', direction: 'desc' }).map(model => model.model_name), ['Beta', 'Alpha', 'Zulu'])
|
||||
assert.deepEqual(input, original)
|
||||
})
|
||||
|
||||
test('paginates grouped models at 50 rows and clamps after filtering', () => {
|
||||
const input = Array.from({ length: 1000 }, (_, index) => ({ model_name: `model-${String(index).padStart(4, '0')}` }))
|
||||
const page = paginateModels(input, 20)
|
||||
|
||||
assert.equal(page.pageSize, 50)
|
||||
assert.equal(page.totalPages, 20)
|
||||
assert.equal(page.items.length, 50)
|
||||
assert.equal(page.items[0].model_name, 'model-0950')
|
||||
assert.equal(paginateModels(input.slice(0, 7), 20).page, 1)
|
||||
})
|
||||
|
||||
const nodes = [
|
||||
{
|
||||
id: 'new',
|
||||
name: 'Atlas',
|
||||
address: '10.0.0.1:50051',
|
||||
node_type: 'backend',
|
||||
status: 'healthy',
|
||||
total_vram: 100,
|
||||
available_vram: 5,
|
||||
total_ram: 200,
|
||||
available_ram: 250,
|
||||
total_disk: 1_000,
|
||||
available_disk: -20,
|
||||
cpu_logical_cores: 8,
|
||||
cpu_usage_percent: 25,
|
||||
cpu_load_1: 1.5,
|
||||
model_count: 3,
|
||||
gpu_vendor: 'NVIDIA',
|
||||
capability: 'nvidia-cuda-13',
|
||||
labels: { zone: 'East', team: 'inference' },
|
||||
},
|
||||
{
|
||||
id: 'old',
|
||||
name: 'Birch',
|
||||
address: 'worker-old.internal',
|
||||
node_type: 'agent',
|
||||
status: 'offline',
|
||||
total_vram: 0,
|
||||
available_vram: 0,
|
||||
total_ram: 100,
|
||||
available_ram: 10,
|
||||
model_count: 0,
|
||||
gpu_vendor: 'unknown',
|
||||
labels: { zone: 'West' },
|
||||
},
|
||||
{
|
||||
id: 'pending',
|
||||
name: 'Cedar',
|
||||
node_type: 'backend',
|
||||
status: 'pending',
|
||||
total_vram: 100,
|
||||
available_vram: 80,
|
||||
total_ram: Number.NaN,
|
||||
available_ram: 10,
|
||||
total_disk: Infinity,
|
||||
available_disk: 2,
|
||||
cpu_logical_cores: 4,
|
||||
cpu_usage_percent: 150,
|
||||
cpu_load_1: -3,
|
||||
labels: {},
|
||||
},
|
||||
]
|
||||
|
||||
test('summarizes mixed worker generations and clamps malformed capacity readings', () => {
|
||||
const summary = summarizeFleet(nodes)
|
||||
|
||||
assert.deepEqual(summary.health, {
|
||||
total: 3,
|
||||
healthy: 1,
|
||||
draining: 0,
|
||||
pending: 1,
|
||||
offline: 1,
|
||||
unhealthy: 0,
|
||||
other: 0,
|
||||
})
|
||||
const { usagePercent: vramUsagePercent, ...vram } = summary.vram
|
||||
assert.deepEqual(vram, {
|
||||
total: 200,
|
||||
used: 115,
|
||||
available: 85,
|
||||
reportingCount: 2,
|
||||
unknownCount: 1,
|
||||
})
|
||||
assert.ok(Math.abs(vramUsagePercent - 57.5) < Number.EPSILON * 100)
|
||||
assert.deepEqual(summary.ram, {
|
||||
total: 300,
|
||||
used: 90,
|
||||
available: 210,
|
||||
usagePercent: 30,
|
||||
reportingCount: 2,
|
||||
unknownCount: 1,
|
||||
})
|
||||
assert.deepEqual(summary.disk, {
|
||||
total: 1_000,
|
||||
used: 1_000,
|
||||
available: 0,
|
||||
usagePercent: 100,
|
||||
reportingCount: 1,
|
||||
unknownCount: 2,
|
||||
})
|
||||
assert.deepEqual(summary.cpu, {
|
||||
totalLogicalCores: 12,
|
||||
busyCoreEquivalents: 6,
|
||||
idleCoreEquivalents: 6,
|
||||
usagePercent: 50,
|
||||
load1: 1.5,
|
||||
reportingCount: 2,
|
||||
unknownCount: 1,
|
||||
})
|
||||
})
|
||||
|
||||
test('deduplicates headline attention while retaining every matching category', () => {
|
||||
const { attention, attentionNodeCount } = summarizeFleet(nodes)
|
||||
|
||||
assert.equal(attentionNodeCount, 3)
|
||||
assert.deepEqual(attention, {
|
||||
pending: ['pending'],
|
||||
offlineOrUnhealthy: ['old'],
|
||||
lowVRAM: ['new'],
|
||||
lowRAM: ['old'],
|
||||
lowDisk: ['new'],
|
||||
})
|
||||
})
|
||||
|
||||
test('does not flag zero-total or invalid-total capacity as exhausted', () => {
|
||||
const summary = summarizeFleet([
|
||||
{ id: 'zero', status: 'healthy', total_vram: 0, available_vram: 0 },
|
||||
{ id: 'bad', status: 'healthy', total_ram: -10, available_ram: 0, total_vram: 8, available_vram: Number.NaN },
|
||||
])
|
||||
|
||||
assert.equal(summary.attentionNodeCount, 0)
|
||||
assert.deepEqual(summary.attention.lowVRAM, [])
|
||||
assert.equal(summary.vram.unknownCount, 2)
|
||||
assert.equal(summary.ram.unknownCount, 2)
|
||||
})
|
||||
|
||||
test('requires finite total and available values before reporting capacity', () => {
|
||||
const summary = summarizeFleet([
|
||||
{ id: 'complete', status: 'healthy', total_vram: 100, available_vram: 120 },
|
||||
{ id: 'missing', status: 'healthy', total_vram: 200 },
|
||||
{ id: 'malformed', status: 'healthy', total_vram: 300, available_vram: '30' },
|
||||
{ id: 'non-finite', status: 'healthy', total_vram: 400, available_vram: Infinity },
|
||||
])
|
||||
|
||||
assert.deepEqual(summary.vram, {
|
||||
total: 100,
|
||||
used: 0,
|
||||
available: 100,
|
||||
usagePercent: 0,
|
||||
reportingCount: 1,
|
||||
unknownCount: 3,
|
||||
})
|
||||
assert.deepEqual(summary.attention.lowVRAM, [])
|
||||
assert.deepEqual(capacityReading(100, -20), { total: 100, used: 100, available: 0, usagePercent: 100 })
|
||||
assert.equal(capacityReading(100, undefined), null)
|
||||
assert.equal(capacityReading(100, Number.NaN), null)
|
||||
})
|
||||
|
||||
test('maps only server-accepted lifecycle states to controls', () => {
|
||||
assert.equal(nodeLifecycleAction('healthy'), 'drain')
|
||||
assert.equal(nodeLifecycleAction('draining'), 'resume')
|
||||
assert.equal(nodeLifecycleAction('pending'), 'approve')
|
||||
for (const status of ['unhealthy', 'offline', 'unknown', '', null, 'HEALTHY']) {
|
||||
assert.equal(nodeLifecycleAction(status), null)
|
||||
}
|
||||
})
|
||||
|
||||
test('requires a complete finite CPU reading before including a node', () => {
|
||||
const summary = summarizeFleet([
|
||||
{ id: 'complete', cpu_logical_cores: 4, cpu_usage_percent: 50, cpu_load_1: 0.5 },
|
||||
{ id: 'missing-load', cpu_logical_cores: 8, cpu_usage_percent: 25 },
|
||||
{ id: 'bad-usage', cpu_logical_cores: 2, cpu_usage_percent: Number.NaN, cpu_load_1: 1 },
|
||||
])
|
||||
|
||||
assert.deepEqual(summary.cpu, {
|
||||
totalLogicalCores: 4,
|
||||
busyCoreEquivalents: 2,
|
||||
idleCoreEquivalents: 2,
|
||||
usagePercent: 50,
|
||||
load1: 0.5,
|
||||
reportingCount: 1,
|
||||
unknownCount: 2,
|
||||
})
|
||||
})
|
||||
|
||||
test('filters without mutation across roster fields and labels case-insensitively', () => {
|
||||
const original = [...nodes]
|
||||
|
||||
assert.deepEqual(filterNodes(nodes, { query: 'CUDA-13', statuses: [], types: [] }).map(node => node.id), ['new'])
|
||||
assert.deepEqual(filterNodes(nodes, { query: 'east', statuses: [], types: [] }).map(node => node.id), ['new'])
|
||||
assert.deepEqual(filterNodes(nodes, { query: '3', statuses: [], types: [] }).map(node => node.id), ['new'])
|
||||
assert.deepEqual(filterNodes(nodes, { query: '', statuses: ['OFFLINE'], types: ['AGENT'] }).map(node => node.id), ['old'])
|
||||
assert.deepEqual(nodes, original)
|
||||
})
|
||||
|
||||
test('sorts stably with node name as the final tie-breaker and leaves input untouched', () => {
|
||||
const input = [
|
||||
{ id: 'z1', name: 'Zulu', model_count: 2 },
|
||||
{ id: 'a1', name: 'Alpha', model_count: 2 },
|
||||
{ id: 'a2', name: 'Alpha', model_count: 2 },
|
||||
{ id: 'b1', name: 'Beta', model_count: 1 },
|
||||
]
|
||||
|
||||
assert.deepEqual(sortNodes(input, { key: 'model_count', direction: 'asc' }).map(node => node.id), ['b1', 'a1', 'a2', 'z1'])
|
||||
assert.deepEqual(sortNodes(input, { key: 'model_count', direction: 'desc' }).map(node => node.id), ['a1', 'a2', 'z1', 'b1'])
|
||||
assert.deepEqual(input.map(node => node.id), ['z1', 'a1', 'a2', 'b1'])
|
||||
})
|
||||
|
||||
test('groups missing labels separately from a real label value equal to unlabelled', () => {
|
||||
const grouped = groupNodes([
|
||||
{ id: 'missing', name: 'Missing', labels: {} },
|
||||
{ id: 'literal', name: 'Literal', labels: { zone: 'unlabelled' } },
|
||||
{ id: 'east', name: 'East', labels: { zone: 'east' } },
|
||||
], 'label:zone')
|
||||
|
||||
assert.equal(grouped.length, 3)
|
||||
assert.equal(new Set(grouped.map(group => group.key)).size, 3)
|
||||
assert.deepEqual(grouped.map(group => ({ label: group.label, ids: group.nodes.map(node => node.id) })), [
|
||||
{ label: 'east', ids: ['east'] },
|
||||
{ label: 'unlabelled', ids: ['literal'] },
|
||||
{ label: 'Unlabelled', ids: ['missing'] },
|
||||
])
|
||||
})
|
||||
|
||||
test('groups by node type and treats none as one group', () => {
|
||||
assert.deepEqual(groupNodes(nodes, 'node_type').map(group => group.label), ['agent', 'backend'])
|
||||
assert.deepEqual(groupNodes(nodes, 'none'), [{ key: 'all', label: 'All nodes', nodes }])
|
||||
})
|
||||
|
||||
test('paginates with a default of 50 rows and clamps the requested page', () => {
|
||||
const input = Array.from({ length: 61 }, (_, index) => ({ id: index + 1 }))
|
||||
|
||||
assert.deepEqual(paginateNodes(input, 99), {
|
||||
page: 2,
|
||||
pageSize: 50,
|
||||
totalItems: 61,
|
||||
totalPages: 2,
|
||||
items: input.slice(50),
|
||||
})
|
||||
assert.deepEqual(paginateNodes([], -4), {
|
||||
page: 1,
|
||||
pageSize: 50,
|
||||
totalItems: 0,
|
||||
totalPages: 1,
|
||||
items: [],
|
||||
})
|
||||
})
|
||||
|
||||
test('runs at most eight operations concurrently and preserves settled result order', async () => {
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const input = Array.from({ length: 25 }, (_, index) => index)
|
||||
|
||||
const results = await runBounded(input, 8, async item => {
|
||||
active += 1
|
||||
peak = Math.max(peak, active)
|
||||
await new Promise(resolve => setTimeout(resolve, (25 - item) % 4))
|
||||
active -= 1
|
||||
if (item === 7) throw new Error('seven failed')
|
||||
return item * 2
|
||||
})
|
||||
|
||||
assert.equal(peak, 8)
|
||||
assert.equal(results.length, 25)
|
||||
assert.deepEqual(results[0], { status: 'fulfilled', value: 0 })
|
||||
assert.equal(results[7].status, 'rejected')
|
||||
assert.equal(results[7].reason.message, 'seven failed')
|
||||
assert.deepEqual(results[24], { status: 'fulfilled', value: 48 })
|
||||
})
|
||||
|
||||
test('rejects a non-positive concurrency limit', async () => {
|
||||
await assert.rejects(runBounded([1], 0, async value => value), RangeError)
|
||||
})
|
||||
@@ -155,6 +155,31 @@ var _ = Describe("heartbeat checkpointing", func() {
|
||||
"rather than movement suppresses nothing at all")
|
||||
})
|
||||
|
||||
It("holds changed CPU readings until the scheduled checkpoint", func() {
|
||||
registry.SetHeartbeatCheckpoint(time.Hour)
|
||||
usage := 25.0
|
||||
load1 := 1.5
|
||||
Expect(registry.Heartbeat(ctx, nodeID, &HeartbeatUpdate{
|
||||
CPUUsagePercent: &usage,
|
||||
CPULoad1: &load1,
|
||||
})).To(Succeed())
|
||||
first := writtenAt()
|
||||
|
||||
usage = 75
|
||||
load1 = 4.5
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
Expect(registry.Heartbeat(ctx, nodeID, &HeartbeatUpdate{
|
||||
CPUUsagePercent: &usage,
|
||||
CPULoad1: &load1,
|
||||
})).To(Succeed())
|
||||
|
||||
var stored BackendNode
|
||||
Expect(db.First(&stored, "id = ?", nodeID).Error).ToNot(HaveOccurred())
|
||||
Expect(stored.LastHeartbeat).To(BeTemporally("==", first))
|
||||
Expect(stored.CPUUsagePercent).To(Equal(25.0))
|
||||
Expect(stored.CPULoad1).To(Equal(1.5))
|
||||
})
|
||||
|
||||
It("suppresses a re-reported total VRAM and GPU vendor that have not changed", func() {
|
||||
registry.SetHeartbeatCheckpoint(time.Hour)
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -35,9 +36,12 @@ type BackendNode struct {
|
||||
// heartbeat (the worker is the source of truth for actual free VRAM); the
|
||||
// reservation is only here to keep two scheduling decisions within the
|
||||
// same heartbeat window from over-committing the same node.
|
||||
ReservedVRAM uint64 `gorm:"column:reserved_vram;default:0" json:"reserved_vram"`
|
||||
TotalRAM uint64 `gorm:"column:total_ram" json:"total_ram"` // Total system RAM in bytes (fallback when no GPU)
|
||||
AvailableRAM uint64 `gorm:"column:available_ram" json:"available_ram"` // Available system RAM in bytes
|
||||
ReservedVRAM uint64 `gorm:"column:reserved_vram;default:0" json:"reserved_vram"`
|
||||
TotalRAM uint64 `gorm:"column:total_ram" json:"total_ram"` // Total system RAM in bytes (fallback when no GPU)
|
||||
AvailableRAM uint64 `gorm:"column:available_ram" json:"available_ram"` // Available system RAM in bytes
|
||||
CPULogicalCores uint64 `gorm:"column:cpu_logical_cores;default:0" json:"cpu_logical_cores"`
|
||||
CPUUsagePercent float64 `gorm:"column:cpu_usage_percent;default:0" json:"cpu_usage_percent"`
|
||||
CPULoad1 float64 `gorm:"column:cpu_load_1;default:0" json:"cpu_load_1"`
|
||||
// TotalDisk / AvailableDisk describe the filesystem that BACKS THE WORKER'S
|
||||
// MODELS DIRECTORY, not the root filesystem: staged weights are written
|
||||
// there, so that is the only mount whose free space decides whether a
|
||||
@@ -107,6 +111,9 @@ const (
|
||||
ColTotalVRAM = "total_vram"
|
||||
ColReservedVRAM = "reserved_vram"
|
||||
ColAvailableRAM = "available_ram"
|
||||
ColCPULogicalCores = "cpu_logical_cores"
|
||||
ColCPUUsagePercent = "cpu_usage_percent"
|
||||
ColCPULoad1 = "cpu_load_1"
|
||||
ColTotalDisk = "total_disk"
|
||||
ColAvailableDisk = "available_disk"
|
||||
ColGPUVendor = "gpu_vendor"
|
||||
@@ -117,6 +124,14 @@ const (
|
||||
ColVRAMBudgetBytes = "vram_budget_bytes"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNodeNotFound reports that a lifecycle transition targeted a missing node.
|
||||
ErrNodeNotFound = gorm.ErrRecordNotFound
|
||||
// ErrNodeStatusConflict reports that a node exists but no longer has the
|
||||
// status required by a conditional lifecycle transition.
|
||||
ErrNodeStatusConflict = errors.New("node status conflict")
|
||||
)
|
||||
|
||||
// NodeModel tracks which models are loaded on which nodes.
|
||||
//
|
||||
// Multiple replicas of the same model on the same node are allowed; each
|
||||
@@ -554,6 +569,8 @@ func capAvailable(reported, ceilingBytes uint64) uint64 {
|
||||
// nodes that were never approved stay in "pending".
|
||||
func (r *NodeRegistry) Register(ctx context.Context, node *BackendNode, autoApprove bool) error {
|
||||
node.LastHeartbeat = time.Now()
|
||||
hasCPUTelemetry := node.CPULogicalCores > 0 || node.CPUUsagePercent != 0 || node.CPULoad1 != 0
|
||||
node.CPUUsagePercent = clampCPUUsage(node.CPUUsagePercent)
|
||||
|
||||
// Try to find existing node by name
|
||||
var existing BackendNode
|
||||
@@ -623,6 +640,20 @@ func (r *NodeRegistry) Register(ctx context.Context, node *BackendNode, autoAppr
|
||||
}).Error; err != nil {
|
||||
return fmt.Errorf("recording disk capacity for node %s: %w", node.Name, err)
|
||||
}
|
||||
// A successful CPU sample always reports logical cores. Use that as the
|
||||
// presence signal so an omitted sample from an older or temporarily
|
||||
// failing worker preserves the last reading, while a real 0% reading is
|
||||
// still force-written despite GORM's struct zero-value suppression.
|
||||
if hasCPUTelemetry {
|
||||
if err := r.db.WithContext(ctx).Model(&BackendNode{}).Where("id = ?", node.ID).
|
||||
Updates(map[string]any{
|
||||
ColCPULogicalCores: node.CPULogicalCores,
|
||||
ColCPUUsagePercent: node.CPUUsagePercent,
|
||||
ColCPULoad1: node.CPULoad1,
|
||||
}).Error; err != nil {
|
||||
return fmt.Errorf("recording CPU telemetry for node %s: %w", node.Name, err)
|
||||
}
|
||||
}
|
||||
// Preserve auth references from existing record.
|
||||
// GORM Updates(struct) skips zero-value fields, so the DB retains
|
||||
// the old auth_user_id/api_key_id but the caller's struct is empty.
|
||||
@@ -724,6 +755,27 @@ func (r *NodeRegistry) setStatus(ctx context.Context, nodeID, status string) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func transitionStatus(db *gorm.DB, nodeID, expectedStatus, nextStatus string) error {
|
||||
result := db.Model(&BackendNode{}).
|
||||
Where("id = ? AND status = ?", nodeID, expectedStatus).
|
||||
Update("status", nextStatus)
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("transitioning node %s from %s to %s: %w", nodeID, expectedStatus, nextStatus, result.Error)
|
||||
}
|
||||
if result.RowsAffected > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var node BackendNode
|
||||
if err := db.Select("id", "status").First(&node, "id = ?", nodeID).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return fmt.Errorf("node %s: %w", nodeID, ErrNodeNotFound)
|
||||
}
|
||||
return fmt.Errorf("checking node %s after conditional transition: %w", nodeID, err)
|
||||
}
|
||||
return fmt.Errorf("node %s has status %s, expected %s: %w", nodeID, node.Status, expectedStatus, ErrNodeStatusConflict)
|
||||
}
|
||||
|
||||
// MarkOffline sets a node to offline status and clears its model records.
|
||||
// Used on graceful shutdown — preserves the node row so re-registration
|
||||
// can restore the previous approval status.
|
||||
@@ -1032,9 +1084,21 @@ type HeartbeatUpdate struct {
|
||||
// AvailableDisk / TotalDisk describe the worker's models filesystem.
|
||||
// Pointers so a worker that cannot read them omits the fields rather than
|
||||
// reporting a zero the scheduler would act on.
|
||||
AvailableDisk *uint64 `json:"available_disk,omitempty"`
|
||||
TotalDisk *uint64 `json:"total_disk,omitempty"`
|
||||
GPUVendor string `json:"gpu_vendor,omitempty"`
|
||||
AvailableDisk *uint64 `json:"available_disk,omitempty"`
|
||||
TotalDisk *uint64 `json:"total_disk,omitempty"`
|
||||
GPUVendor string `json:"gpu_vendor,omitempty"`
|
||||
CPUUsagePercent *float64 `json:"cpu_usage_percent,omitempty"`
|
||||
CPULoad1 *float64 `json:"cpu_load_1,omitempty"`
|
||||
}
|
||||
|
||||
func clampCPUUsage(usage float64) float64 {
|
||||
if math.IsNaN(usage) || usage < 0 {
|
||||
return 0
|
||||
}
|
||||
if usage > 100 {
|
||||
return 100
|
||||
}
|
||||
return usage
|
||||
}
|
||||
|
||||
// Heartbeat updates the heartbeat timestamp and status for a node.
|
||||
@@ -1090,6 +1154,12 @@ func (r *NodeRegistry) Heartbeat(ctx context.Context, nodeID string, update *Hea
|
||||
if update.GPUVendor != "" {
|
||||
updates[ColGPUVendor] = update.GPUVendor
|
||||
}
|
||||
if update.CPUUsagePercent != nil {
|
||||
updates[ColCPUUsagePercent] = clampCPUUsage(*update.CPUUsagePercent)
|
||||
}
|
||||
if update.CPULoad1 != nil {
|
||||
updates[ColCPULoad1] = *update.CPULoad1
|
||||
}
|
||||
}
|
||||
|
||||
// Recorded with the ceiling this write actually resolved, so the snapshot
|
||||
@@ -1248,30 +1318,28 @@ func (r *NodeRegistry) MarkHealthy(ctx context.Context, nodeID string) error {
|
||||
// observable effect is that the per-call IncrementInFlight bookkeeping logs a
|
||||
// non-fatal warning, which is acceptable for a drain.
|
||||
func (r *NodeRegistry) MarkDraining(ctx context.Context, nodeID string) error {
|
||||
if err := r.setStatus(ctx, nodeID, StatusDraining); err != nil {
|
||||
return err
|
||||
}
|
||||
// Capture the distinct models and run the bulk delete inside a single
|
||||
// transaction so the set of fired hooks equals exactly the set of rows
|
||||
// deleted: a SetNodeModel landing between the capture and the delete can no
|
||||
// longer be deleted without its hook firing (no interleaving gap). The
|
||||
// status flip above is a separate, pre-existing operation and stays outside
|
||||
// this transaction. Fire hooks only after commit so a rollback does not
|
||||
// invalidate the index for a removal that did not persist.
|
||||
var removedModels []string
|
||||
if err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := transitionStatus(tx, nodeID, StatusHealthy, StatusDraining); err != nil {
|
||||
return err
|
||||
}
|
||||
removedModels = r.nodeModelNames(ctx, tx, nodeID)
|
||||
return tx.Where("node_id = ?", nodeID).Delete(&NodeModel{}).Error
|
||||
}); err != nil {
|
||||
xlog.Warn("Failed to clear model records on draining", "node", nodeID, "error", err)
|
||||
} else {
|
||||
for _, m := range removedModels {
|
||||
r.fireReplicaRemoved(m, nodeID, -1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
for _, m := range removedModels {
|
||||
r.fireReplicaRemoved(m, nodeID, -1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResumeNode transitions a draining node back to healthy without allowing
|
||||
// pending approval or a concurrent health-state change to be overwritten.
|
||||
func (r *NodeRegistry) ResumeNode(ctx context.Context, nodeID string) error {
|
||||
return transitionStatus(r.db.WithContext(ctx), nodeID, StatusDraining, StatusHealthy)
|
||||
}
|
||||
|
||||
// FindStaleNodes returns nodes that haven't sent a heartbeat within the given threshold.
|
||||
// Excludes unhealthy, offline, and pending nodes since they're not actively participating.
|
||||
func (r *NodeRegistry) FindStaleNodes(ctx context.Context, threshold time.Duration) ([]BackendNode, error) {
|
||||
|
||||
@@ -41,6 +41,20 @@ var _ = Describe("NodeRegistry", func() {
|
||||
}
|
||||
|
||||
Describe("Register", func() {
|
||||
It("persists CPU telemetry and clamps utilization", func() {
|
||||
node := makeNode("cpu-worker", "10.0.0.3:50051", 0)
|
||||
node.CPULogicalCores = 16
|
||||
node.CPUUsagePercent = 140
|
||||
node.CPULoad1 = 2.75
|
||||
|
||||
Expect(registry.Register(context.Background(), node, true)).To(Succeed())
|
||||
fetched, err := registry.GetByName(context.Background(), "cpu-worker")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fetched.CPULogicalCores).To(Equal(uint64(16)))
|
||||
Expect(fetched.CPUUsagePercent).To(Equal(float64(100)))
|
||||
Expect(fetched.CPULoad1).To(Equal(2.75))
|
||||
})
|
||||
|
||||
It("sets StatusPending when autoApprove is false", func() {
|
||||
node := makeNode("worker-1", "10.0.0.1:50051", 8_000_000_000)
|
||||
Expect(registry.Register(context.Background(), node, false)).To(Succeed())
|
||||
@@ -59,6 +73,21 @@ var _ = Describe("NodeRegistry", func() {
|
||||
})
|
||||
|
||||
Describe("Re-registration", func() {
|
||||
It("persists CPU utilization clamped to zero", func() {
|
||||
first := makeNode("cpu-reregister", "10.0.0.8:50051", 0)
|
||||
first.CPUUsagePercent = 65
|
||||
Expect(registry.Register(context.Background(), first, true)).To(Succeed())
|
||||
|
||||
second := makeNode("cpu-reregister", "10.0.0.8:50051", 0)
|
||||
second.CPULogicalCores = 16
|
||||
second.CPUUsagePercent = -5
|
||||
Expect(registry.Register(context.Background(), second, true)).To(Succeed())
|
||||
|
||||
fetched, err := registry.GetByName(context.Background(), "cpu-reregister")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fetched.CPUUsagePercent).To(Equal(float64(0)))
|
||||
})
|
||||
|
||||
It("keeps a pending node pending on re-register with autoApprove=false", func() {
|
||||
node := makeNode("re-pending", "10.0.0.3:50051", 4_000_000_000)
|
||||
Expect(registry.Register(context.Background(), node, false)).To(Succeed())
|
||||
@@ -578,6 +607,62 @@ var _ = Describe("NodeRegistry", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Conditional lifecycle transitions", func() {
|
||||
It("accepts healthy to draining to healthy transitions", func() {
|
||||
node := makeNode("conditional-roundtrip", "10.0.0.61:50051", 8_000_000_000)
|
||||
Expect(registry.Register(context.Background(), node, true)).To(Succeed())
|
||||
|
||||
Expect(registry.MarkDraining(context.Background(), node.ID)).To(Succeed())
|
||||
fetched, err := registry.Get(context.Background(), node.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fetched.Status).To(Equal(StatusDraining))
|
||||
|
||||
Expect(registry.ResumeNode(context.Background(), node.ID)).To(Succeed())
|
||||
fetched, err = registry.Get(context.Background(), node.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fetched.Status).To(Equal(StatusHealthy))
|
||||
})
|
||||
|
||||
It("rejects pending drain and resume without changing status", func() {
|
||||
node := makeNode("conditional-pending", "10.0.0.62:50051", 8_000_000_000)
|
||||
Expect(registry.Register(context.Background(), node, false)).To(Succeed())
|
||||
|
||||
Expect(errors.Is(registry.MarkDraining(context.Background(), node.ID), ErrNodeStatusConflict)).To(BeTrue())
|
||||
Expect(errors.Is(registry.ResumeNode(context.Background(), node.ID), ErrNodeStatusConflict)).To(BeTrue())
|
||||
fetched, err := registry.Get(context.Background(), node.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fetched.Status).To(Equal(StatusPending))
|
||||
})
|
||||
|
||||
It("distinguishes a missing node from an expected-status mismatch", func() {
|
||||
err := registry.MarkDraining(context.Background(), "missing")
|
||||
Expect(errors.Is(err, ErrNodeNotFound)).To(BeTrue())
|
||||
|
||||
node := makeNode("conditional-mismatch", "10.0.0.63:50051", 8_000_000_000)
|
||||
Expect(registry.Register(context.Background(), node, true)).To(Succeed())
|
||||
Expect(registry.SetNodeModel(context.Background(), node.ID, "race-model", 0, "loaded", node.Address, 0)).To(Succeed())
|
||||
before, err := registry.Get(context.Background(), node.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(before.Status).To(Equal(StatusHealthy))
|
||||
removedHooks := 0
|
||||
registry.AddReplicaRemovedHook(func(string, string, int) { removedHooks++ })
|
||||
|
||||
// Simulate a health update landing after the caller observed healthy
|
||||
// but before its conditional drain reaches persistence.
|
||||
Expect(registry.MarkUnhealthy(context.Background(), node.ID)).To(Succeed())
|
||||
|
||||
err = registry.MarkDraining(context.Background(), node.ID)
|
||||
Expect(errors.Is(err, ErrNodeStatusConflict)).To(BeTrue())
|
||||
fetched, getErr := registry.Get(context.Background(), node.ID)
|
||||
Expect(getErr).ToNot(HaveOccurred())
|
||||
Expect(fetched.Status).To(Equal(StatusUnhealthy))
|
||||
models, modelsErr := registry.GetNodeModels(context.Background(), node.ID)
|
||||
Expect(modelsErr).ToNot(HaveOccurred())
|
||||
Expect(models).To(HaveLen(1))
|
||||
Expect(removedHooks).To(BeZero())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("NodeLabel CRUD", func() {
|
||||
It("sets and retrieves labels for a node", func() {
|
||||
node := makeNode("label-node", "10.0.0.70:50051", 8_000_000_000)
|
||||
|
||||
@@ -172,7 +172,12 @@ func (a *RemoteUnloaderAdapter) UnloadRemoteModelContext(ctx context.Context, mo
|
||||
}
|
||||
|
||||
var unloadErr error
|
||||
seenNodeIDs := make(map[string]struct{}, len(nodes))
|
||||
for _, node := range nodes {
|
||||
if _, seen := seenNodeIDs[node.ID]; seen {
|
||||
continue
|
||||
}
|
||||
seenNodeIDs[node.ID] = struct{}{}
|
||||
xlog.Info("Sending NATS backend.stop to node", "model", modelName, "node", node.Name, "nodeID", node.ID, "force", force)
|
||||
if err := a.stopBackend(node.ID, modelName, force); err != nil {
|
||||
xlog.Warn("Failed to send backend.stop", "node", node.Name, "error", err)
|
||||
|
||||
@@ -214,6 +214,23 @@ var _ = Describe("RemoteUnloaderAdapter", func() {
|
||||
Expect(locator.removedPairs[1]).To(Equal(modelNodePair{"node-2", "llama"}))
|
||||
})
|
||||
|
||||
It("stops each node once when the registry returns multiple replicas", func() {
|
||||
locator.nodes = []BackendNode{
|
||||
{ID: "node-1", Name: "worker-1"},
|
||||
{ID: "node-1", Name: "worker-1"},
|
||||
{ID: "node-2", Name: "worker-2"},
|
||||
}
|
||||
|
||||
Expect(adapter.UnloadRemoteModel("llama")).To(Succeed())
|
||||
Expect(mc.requestCalls).To(HaveLen(2))
|
||||
Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1")))
|
||||
Expect(mc.requestCalls[1].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-2")))
|
||||
Expect(locator.removedPairs).To(ConsistOf(
|
||||
modelNodePair{"node-1", "llama"},
|
||||
modelNodePair{"node-2", "llama"},
|
||||
))
|
||||
})
|
||||
|
||||
It("continues when one node fails", func() {
|
||||
locator.nodes = []BackendNode{
|
||||
{ID: "node-fail", Name: "worker-fail"},
|
||||
|
||||
@@ -3,6 +3,7 @@ package worker
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
@@ -17,8 +18,19 @@ var (
|
||||
totalAvailableVRAM = xsysinfo.TotalAvailableVRAM
|
||||
getGPUAggregateInfo = xsysinfo.GetGPUAggregateInfo
|
||||
getSystemRAMInfo = xsysinfo.GetSystemRAMInfo
|
||||
getCPUInfo = xsysinfo.GetCPUInfo
|
||||
)
|
||||
|
||||
func clampCPUUsage(usage float64) float64 {
|
||||
if math.IsNaN(usage) || usage < 0 {
|
||||
return 0
|
||||
}
|
||||
if usage > 100 {
|
||||
return 100
|
||||
}
|
||||
return usage
|
||||
}
|
||||
|
||||
// effectiveBasePort returns the port used as base for gRPC backend processes.
|
||||
// Priority: Addr port → ServeAddr port → 50051
|
||||
func (cfg *Config) effectiveBasePort() int {
|
||||
@@ -198,6 +210,14 @@ func (cfg *Config) registrationBody() map[string]any {
|
||||
body["token"] = cfg.RegistrationToken
|
||||
}
|
||||
|
||||
if cpuInfo, err := getCPUInfo(); err != nil {
|
||||
xlog.Debug("Failed to sample worker CPU for registration", "error", err)
|
||||
} else {
|
||||
body["cpu_logical_cores"] = cpuInfo.LogicalCores
|
||||
body["cpu_usage_percent"] = clampCPUUsage(cpuInfo.UsagePercent)
|
||||
body["cpu_load_1"] = cpuInfo.Load1
|
||||
}
|
||||
|
||||
// Parse and add static node labels. Always include the auto-label
|
||||
// `node.replica-slots=N` so AND-selectors in ModelSchedulingConfig can
|
||||
// target high-capacity nodes (e.g. {"node.replica-slots":"4"}).
|
||||
@@ -249,5 +269,12 @@ func (cfg *Config) heartbeatBody() map[string]any {
|
||||
body["total_disk"] = diskInfo.Total
|
||||
body["available_disk"] = diskInfo.Available
|
||||
}
|
||||
|
||||
if cpuInfo, err := getCPUInfo(); err != nil {
|
||||
xlog.Debug("Failed to sample worker CPU for heartbeat", "error", err)
|
||||
} else {
|
||||
body["cpu_usage_percent"] = clampCPUUsage(cpuInfo.UsagePercent)
|
||||
body["cpu_load_1"] = cpuInfo.Load1
|
||||
}
|
||||
return body
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
@@ -12,13 +14,52 @@ var _ = Describe("Worker registration body", func() {
|
||||
originalTotalAvailableVRAM := totalAvailableVRAM
|
||||
originalGetGPUAggregateInfo := getGPUAggregateInfo
|
||||
originalGetSystemRAMInfo := getSystemRAMInfo
|
||||
originalGetCPUInfo := getCPUInfo
|
||||
DeferCleanup(func() {
|
||||
totalAvailableVRAM = originalTotalAvailableVRAM
|
||||
getGPUAggregateInfo = originalGetGPUAggregateInfo
|
||||
getSystemRAMInfo = originalGetSystemRAMInfo
|
||||
getCPUInfo = originalGetCPUInfo
|
||||
})
|
||||
})
|
||||
|
||||
It("reports CPU telemetry on registration and clamps utilization", func() {
|
||||
getCPUInfo = func() (*xsysinfo.CPUInfo, error) {
|
||||
return &xsysinfo.CPUInfo{LogicalCores: 24, UsagePercent: 120, Load1: 3.5}, nil
|
||||
}
|
||||
|
||||
body := (&Config{}).registrationBody()
|
||||
|
||||
Expect(body["cpu_logical_cores"]).To(Equal(uint64(24)))
|
||||
Expect(body["cpu_usage_percent"]).To(Equal(float64(100)))
|
||||
Expect(body["cpu_load_1"]).To(Equal(3.5))
|
||||
})
|
||||
|
||||
It("reports dynamic CPU telemetry on heartbeats", func() {
|
||||
getCPUInfo = func() (*xsysinfo.CPUInfo, error) {
|
||||
return &xsysinfo.CPUInfo{LogicalCores: 24, UsagePercent: -4, Load1: 1.25}, nil
|
||||
}
|
||||
|
||||
body := (&Config{}).heartbeatBody()
|
||||
|
||||
Expect(body["cpu_usage_percent"]).To(Equal(float64(0)))
|
||||
Expect(body["cpu_load_1"]).To(Equal(1.25))
|
||||
Expect(body).ToNot(HaveKey("cpu_logical_cores"))
|
||||
})
|
||||
|
||||
It("omits all CPU fields when sampling fails", func() {
|
||||
getCPUInfo = func() (*xsysinfo.CPUInfo, error) { return nil, errors.New("sampling failed") }
|
||||
|
||||
registration := (&Config{}).registrationBody()
|
||||
heartbeat := (&Config{}).heartbeatBody()
|
||||
|
||||
for _, body := range []map[string]any{registration, heartbeat} {
|
||||
Expect(body).ToNot(HaveKey("cpu_logical_cores"))
|
||||
Expect(body).ToNot(HaveKey("cpu_usage_percent"))
|
||||
Expect(body).ToNot(HaveKey("cpu_load_1"))
|
||||
}
|
||||
})
|
||||
|
||||
It("includes the VRAM budget in the registration body when set", func() {
|
||||
cfg := &Config{VRAMBudget: "80%"}
|
||||
body := cfg.registrationBody()
|
||||
|
||||
@@ -434,6 +434,33 @@ from VRAM on every registration and heartbeat. On unified-memory nodes, the
|
||||
available RAM and available VRAM values should therefore track each other
|
||||
closely; on discrete-GPU nodes they can change independently.
|
||||
|
||||
### CPU telemetry
|
||||
|
||||
Backend workers report host-wide CPU telemetry in `GET /api/nodes` and
|
||||
`GET /api/nodes/:id`:
|
||||
|
||||
| Field | Meaning |
|
||||
|-------|---------|
|
||||
| `cpu_logical_cores` | Logical processor count, sampled at registration |
|
||||
| `cpu_usage_percent` | Utilization across the whole host, clamped to `0..100` |
|
||||
| `cpu_load_1` | One-minute system load average |
|
||||
|
||||
Utilization and load are sampled at registration and again at each worker
|
||||
heartbeat (every 10 seconds by default). The frontend persists heartbeat
|
||||
samples on the normal heartbeat checkpoint cadence; CPU movement alone does
|
||||
not force an extra database write. If a sample fails, the worker omits all CPU
|
||||
fields and the frontend keeps the last successful reading.
|
||||
|
||||
Workers from releases that predate CPU reporting remain compatible. Their
|
||||
`cpu_logical_cores` value is zero, which means unknown rather than a zero-core
|
||||
machine. Fleet capacity excludes those workers from CPU totals and reports
|
||||
them as unknown. The dashboard derives available CPU as idle logical-core
|
||||
equivalents:
|
||||
|
||||
```
|
||||
idle cores = cpu_logical_cores * (1 - cpu_usage_percent / 100)
|
||||
```
|
||||
|
||||
### Node Labels
|
||||
|
||||
Workers can declare labels at startup for scheduling constraints:
|
||||
@@ -487,6 +514,7 @@ Used by the WebUI and admin API consumers. Requires admin authentication.
|
||||
| `GET` | `/api/nodes` | List all registered workers |
|
||||
| `GET` | `/api/nodes/:id` | Get a single worker by ID |
|
||||
| `GET` | `/api/nodes/:id/models` | List models loaded on a worker |
|
||||
| `GET` | `/api/nodes/models` | List loaded model replicas on healthy workers |
|
||||
| `DELETE` | `/api/nodes/:id` | Admin-delete a worker |
|
||||
| `POST` | `/api/nodes/:id/drain` | Admin-drain a worker |
|
||||
| `POST` | `/api/nodes/:id/approve` | Approve a pending worker node |
|
||||
@@ -498,7 +526,17 @@ Used by the WebUI and admin API consumers. Requires admin authentication.
|
||||
| `PUT` | `/api/nodes/:id/vram-budget` | Set a VRAM budget for a worker (`{"value":"80%"}`) |
|
||||
| `DELETE` | `/api/nodes/:id/vram-budget` | Clear a worker's VRAM budget (revert to all detected VRAM) |
|
||||
|
||||
The **Nodes** page in the React WebUI provides a visual overview of all registered workers, their statuses, and loaded models. The page opens with a one-line **cluster pulse** summarising node health and an **attention callout** that surfaces nodes needing action (for example pending approvals). Below that, a roster of **node panels** lists each worker with its inline model chips (no expand click needed), filtered by an **All / Backend / Agent** segmented control. Selecting a panel opens a dedicated **node detail page** at `/app/nodes/:id` with per-node metrics, models, and backend actions. Model scheduling lives on its own **Scheduling** page (separate nav item), not as a tab on the Nodes page.
|
||||
The **Nodes** page in the React WebUI is a fleet operations dashboard. Its health band and VRAM, RAM, CPU, and models-disk gauges aggregate the single `GET /api/nodes` response and identify how many workers do not report each metric. The attention queue isolates pending, impaired, or low-capacity workers without double-counting the headline affected-node total.
|
||||
|
||||
The fleet table supports search, status and type filters, label or type grouping, sortable columns, and selection across filters. It renders 50 workers at a time and bulk drain, resume, and remove operations run with bounded concurrency, so the page remains usable for fleets with thousands of registrations. Selecting the visible page or a group does not discard selections elsewhere; selections are removed only when a later poll confirms the worker no longer exists.
|
||||
|
||||
Selecting a row opens an in-context inspector with health, labels, capacity, model activity, and heartbeat details. Backend inventory is fetched only for the open inspector. The inspector links to the dedicated node detail page at `/app/nodes/:id`, where model, backend, label, capacity, CPU utilization and load, and models-disk management remain available. Model scheduling lives on its own **Scheduling** page.
|
||||
|
||||
The workbench's **Running models** tab shows the current loaded replicas on healthy workers. It stays lazy: opening the Nodes page does not query model inventory, and the first activation makes one controller database request that is retained until the page is left. The view groups replicas by model, reports their worker spread, active requests, backend types, and most recent use, and renders 50 models per page for large fleets. Loading, empty, and query-failure states are shown in place; a failed query can be retried.
|
||||
|
||||
Use a model row's actions menu to stop that model across the fleet. LocalAI sends one controller shutdown request for the model, which stops all loaded placements; the browser does not contact workers individually. The dashboard refreshes the running-model inventory after both successful and failed shutdown attempts because a failed request can still have stopped some replicas.
|
||||
|
||||
Opening a model reveals its replica placement without another request. Replicas on the same worker remain individually visible with their process addresses and workload. From there, select a known worker to move into its node inspector, then return to the model with **Back to model**. That worker transition is the only point in this flow that requests backend inventory, preserving the Nodes page's no-prefetch behavior.
|
||||
|
||||
### Model sizing in the WebUI
|
||||
|
||||
@@ -693,6 +731,10 @@ without waiting:
|
||||
- a free VRAM, free RAM or free disk reading that has moved more than 256 MiB, because
|
||||
the scheduler places against those figures
|
||||
|
||||
CPU utilization and load follow the scheduled checkpoint instead of making a
|
||||
heartbeat material. They are dashboard observations and do not affect model
|
||||
placement, so persisting every fluctuation would defeat write suppression.
|
||||
|
||||
Every figure is compared against the value **last written**, not against the previous
|
||||
beat. A worker reports its disk capacity on every single beat, so testing whether a
|
||||
field is merely *present* would make every real beat look like a change and suppress
|
||||
|
||||
@@ -5,7 +5,7 @@ weight = 20
|
||||
url = "/features/backend-monitor/"
|
||||
+++
|
||||
|
||||
LocalAI provides endpoints to monitor and manage running backends. The `/backend/monitor` endpoint reports the status and resource usage of loaded models, `/backend/load` pre-loads a model into memory, and `/backend/shutdown` allows stopping a model's backend process.
|
||||
LocalAI provides endpoints to monitor and manage running backends. The `/backend/monitor` endpoint reports the status and resource usage of loaded models, `/backend/load` pre-loads a model into memory, and `/backend/shutdown` allows stopping a model's backend processes. In distributed mode, a named shutdown stops both a process local to the controller and every placement registered on workers.
|
||||
|
||||
All three are admin-only.
|
||||
|
||||
|
||||
+2
-2
@@ -28,8 +28,8 @@ type ModelUnloadHook func(modelName string)
|
||||
|
||||
// RemoteModelUnloader handles unloading models from remote backend nodes.
|
||||
// In distributed mode, this is implemented by the SmartRouter.
|
||||
// When ShutdownModel is called for a model with no local process,
|
||||
// RemoteModelUnloader.UnloadRemoteModel is called to tell the remote node to free it.
|
||||
// ShutdownModel calls the remote unloader even when a local process exists so
|
||||
// one request stops every placement in a mixed local and distributed fleet.
|
||||
type RemoteModelUnloader interface {
|
||||
UnloadRemoteModel(modelName string) error
|
||||
}
|
||||
|
||||
+18
-7
@@ -170,22 +170,33 @@ func (ml *ModelLoader) deleteProcess(ctx context.Context, s string, force bool)
|
||||
// Mark the stop as intentional so the exit-watcher logs it as an
|
||||
// expected stop, not a crash (signal-terminated children report -1).
|
||||
ml.stoppingProcs.Store(process, struct{}{})
|
||||
err := process.Stop()
|
||||
if err != nil {
|
||||
var localErr error
|
||||
if err := process.Stop(); err != nil {
|
||||
xlog.Error("(deleteProcess) error while deleting process", "error", err, "model", s)
|
||||
if !process.IsAlive() {
|
||||
// A concurrently crashed/already-reaped process can no longer own
|
||||
// resources even if Stop could not read or signal its PID.
|
||||
store.Delete(s)
|
||||
ml.cleanupProcessRuntime(process)
|
||||
return nil
|
||||
} else {
|
||||
localErr = err
|
||||
}
|
||||
return err
|
||||
} else {
|
||||
store.Delete(s)
|
||||
ml.cleanupProcessRuntime(process)
|
||||
}
|
||||
|
||||
store.Delete(s)
|
||||
ml.cleanupProcessRuntime(process)
|
||||
return nil
|
||||
// A model can be resident on this frontend and on workers at the same
|
||||
// time. Always attempt the remote half after the local half so a failure in
|
||||
// either location does not leave the other placements running.
|
||||
var remoteErr error
|
||||
if remoteUnloader != nil {
|
||||
remoteErr = unloadRemote(ctx, remoteUnloader, s, force)
|
||||
if remoteErr != nil {
|
||||
remoteErr = fmt.Errorf("unloading remote placements for model %q: %w", s, remoteErr)
|
||||
}
|
||||
}
|
||||
return errors.Join(localErr, remoteErr)
|
||||
}
|
||||
func (ml *ModelLoader) StopGRPC(filter GRPCProcessFilter) error {
|
||||
var err error = nil
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
process "github.com/mudler/go-processmanager"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -77,6 +78,50 @@ var _ = Describe("ShutdownModel in distributed mode", func() {
|
||||
"stopping a model that is running on a worker must succeed, not report 'model not found'")
|
||||
})
|
||||
|
||||
DescribeTable("stops mixed local and remote placements",
|
||||
func(remoteErr error) {
|
||||
unloader.unloadErr = remoteErr
|
||||
modelLoader.SetRemoteUnloader(unloader)
|
||||
|
||||
localProcess := process.New(
|
||||
process.WithTemporaryStateDir(),
|
||||
process.WithName("/bin/sleep"),
|
||||
process.WithArgs("300"),
|
||||
)
|
||||
Expect(localProcess.Run()).To(Succeed())
|
||||
DeferCleanup(func() {
|
||||
if localProcess.IsAlive() {
|
||||
_ = localProcess.Stop()
|
||||
}
|
||||
})
|
||||
|
||||
_, err := modelLoader.LoadModel("mixed", "mixed", func(_, _, _ string) (*model.Model, error) {
|
||||
return model.NewModel("mixed", "local", localProcess), nil
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
var hookCalls int
|
||||
modelLoader.OnModelUnload(func(modelName string) {
|
||||
Expect(modelName).To(Equal("mixed"))
|
||||
hookCalls++
|
||||
})
|
||||
|
||||
err = modelLoader.ShutdownModelForce("mixed")
|
||||
|
||||
Expect(localProcess.IsAlive()).To(BeFalse())
|
||||
Expect(modelLoader.ListLoadedModels()).To(BeEmpty())
|
||||
Expect(unloader.called).To(ConsistOf("mixed"))
|
||||
Expect(hookCalls).To(Equal(1))
|
||||
if remoteErr == nil {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
return
|
||||
}
|
||||
Expect(errors.Is(err, remoteErr)).To(BeTrue())
|
||||
},
|
||||
Entry("when every placement stops", nil),
|
||||
Entry("when local stop succeeds but a remote placement fails", errors.New("worker unreachable")),
|
||||
)
|
||||
|
||||
It("reports not-found only after the registry confirms no node has it", func() {
|
||||
unloader.present = false
|
||||
modelLoader.SetRemoteUnloader(unloader)
|
||||
|
||||
@@ -1,14 +1,54 @@
|
||||
package xsysinfo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
"github.com/jaypipes/ghw"
|
||||
"github.com/klauspost/cpuid/v2"
|
||||
"github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/shirou/gopsutil/v3/load"
|
||||
)
|
||||
|
||||
// CPUInfo describes host-wide CPU capacity and current utilization.
|
||||
type CPUInfo struct {
|
||||
LogicalCores uint64
|
||||
UsagePercent float64
|
||||
Load1 float64
|
||||
}
|
||||
|
||||
// GetCPUInfo samples host-wide CPU telemetry.
|
||||
func GetCPUInfo() (*CPUInfo, error) {
|
||||
logicalCores, err := cpu.Counts(true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("counting logical CPU cores: %w", err)
|
||||
}
|
||||
if logicalCores < 1 {
|
||||
return nil, fmt.Errorf("counting logical CPU cores: no cores reported")
|
||||
}
|
||||
|
||||
usage, err := cpu.Percent(0, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sampling CPU utilization: %w", err)
|
||||
}
|
||||
if len(usage) != 1 {
|
||||
return nil, fmt.Errorf("sampling CPU utilization: expected one aggregate reading, got %d", len(usage))
|
||||
}
|
||||
|
||||
loadAverage, err := load.Avg()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sampling CPU load: %w", err)
|
||||
}
|
||||
|
||||
return &CPUInfo{
|
||||
LogicalCores: uint64(logicalCores),
|
||||
UsagePercent: usage[0],
|
||||
Load1: loadAverage.Load1,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func CPUCapabilities() ([]string, error) {
|
||||
cpu, err := ghw.CPU()
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package xsysinfo
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("CPU telemetry", func() {
|
||||
It("reports logical cores, utilization, and one-minute load", func() {
|
||||
info, err := GetCPUInfo()
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(info.LogicalCores).To(BeNumerically(">", 0))
|
||||
Expect(info.UsagePercent).To(BeNumerically(">=", 0))
|
||||
Expect(info.UsagePercent).To(BeNumerically("<=", 100))
|
||||
Expect(math.IsNaN(info.Load1)).To(BeFalse())
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user