From 80e3240f2d7ba3a14d301df77ec74bf449661e40 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:16:50 +0200 Subject: [PATCH] feat(distributed): key scheduling rules by a model alias (#11771) Node placement and replica rules could only name a model, so an operator who pinned "llama3" to the GPU tier had to rewrite the rule whenever a different model took over that job. An alias already gives a stable name for whichever model serves it, and a rule on that name makes it a deployment slot: repoint the alias and the placement follows. A rule keeps the name the operator chose. Reads resolve that name through the config loader to the model the rule governs, so the reconciler counts, schedules and trims replicas of the target, and the router finds an alias-keyed rule from the target it is already routing. An alias that resolves to nothing governs nothing loadable, so the reconciler skips it and the write paths refuse it. A replica is shared by every name that resolves to it, so only one rule can decide where it runs. The REST and MCP write paths reject a rule whose target another rule already governs. A pair that arrives some other way, such as a seed file or an alias repointed onto a model that already has a rule, resolves in favour of the rule named after the model itself and then the oldest, and the rest are listed as shadowed. The eviction guard is the exception: it matches rules to replicas in raw SQL inside a locking transaction and cannot resolve an alias. It reads a stored target that the reconciler refreshes each tick, and falls back to the rule's own name when that target is empty. Assisted-by: Claude:claude-opus-5 golangci-lint eslint Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/application/distributed.go | 9 + core/config/model_config_loader.go | 20 ++ core/config/model_config_loader_test.go | 54 ++++ core/http/endpoints/localai/nodes.go | 15 ++ .../localai/nodes_scheduling_alias_test.go | 125 ++++++++++ core/http/react-ui/e2e/scheduling.spec.js | 69 ++++++ core/http/react-ui/src/App.css | 25 ++ .../src/components/SearchableModelSelect.jsx | 17 +- core/http/react-ui/src/pages/Scheduling.jsx | 66 ++++- core/services/nodes/alias_scheduling.go | 230 ++++++++++++++++++ core/services/nodes/alias_scheduling_test.go | 196 +++++++++++++++ core/services/nodes/interfaces.go | 1 + core/services/nodes/model_router_test.go | 3 + core/services/nodes/reconciler.go | 63 +++-- core/services/nodes/reconciler_alias_test.go | 133 ++++++++++ core/services/nodes/registry.go | 62 ++++- core/services/nodes/router.go | 25 +- .../nodes/router_eviction_alias_test.go | 104 ++++++++ core/services/nodes/router_test.go | 38 ++- docs/content/features/distributed-mode.md | 34 +++ docs/content/features/model-aliases.md | 17 ++ pkg/mcp/localaitools/dto.go | 8 +- pkg/mcp/localaitools/inproc/client.go | 9 + pkg/mcp/localaitools/scheduling.go | 2 + 24 files changed, 1291 insertions(+), 34 deletions(-) create mode 100644 core/http/endpoints/localai/nodes_scheduling_alias_test.go create mode 100644 core/services/nodes/alias_scheduling.go create mode 100644 core/services/nodes/alias_scheduling_test.go create mode 100644 core/services/nodes/reconciler_alias_test.go create mode 100644 core/services/nodes/router_eviction_alias_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 8389c5c9f..b7dc0bf91 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -162,6 +162,15 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("Node registry initialized") + // Let scheduling rules be keyed by a model alias. The registry resolves a + // rule's name through the config loader to find the model it governs, so an + // operator can pin placement to a stable name like "production" and have it + // follow the alias when the alias is repointed. Wired before the seed below + // and before the reconciler starts, so the first tick already resolves. + if configLoader != nil { + registry.SetAliasResolver(configLoader) + } + // Seed declarative per-model scheduling config (LOCALAI_MODEL_SCHEDULING / // LOCALAI_MODEL_SCHEDULING_CONFIG). Authoritative: overwrites matching models // on every boot. Runs before the reconciler starts so the first tick already diff --git a/core/config/model_config_loader.go b/core/config/model_config_loader.go index b91449ff0..9120cc268 100644 --- a/core/config/model_config_loader.go +++ b/core/config/model_config_loader.go @@ -441,6 +441,26 @@ func (bcl *ModelConfigLoader) ResolveAlias(cfg *ModelConfig) (*ModelConfig, bool return &target, true, nil } +// ResolveAliasName maps a model name to the name of the model that actually +// serves it: an alias resolves to its target, anything else resolves to +// itself. The second return reports whether name was an alias. +// +// Unlike ResolveAlias this never errors. A name with no config (a rule may be +// authored before the model is installed), a dangling alias, and a chained +// alias all resolve to themselves, so callers keep a usable name that simply +// has no model behind it rather than silently governing a different model. +func (bcl *ModelConfigLoader) ResolveAliasName(name string) (string, bool) { + cfg, exists := bcl.GetModelConfig(name) + if !exists || !cfg.IsAlias() { + return name, false + } + target, exists := bcl.GetModelConfig(cfg.Alias) + if !exists || target.IsAlias() { + return name, true + } + return target.Name, true +} + // ValidateAliasTarget checks an alias config's target at create/swap time: // the target must exist, must not be an alias, and must not be disabled. // Returns nil for non-alias configs. diff --git a/core/config/model_config_loader_test.go b/core/config/model_config_loader_test.go index 87807deec..d654226ef 100644 --- a/core/config/model_config_loader_test.go +++ b/core/config/model_config_loader_test.go @@ -314,3 +314,57 @@ var _ = Describe("ModelConfigLoader alias resolution", func() { Expect(loader.ValidateAliasTarget(&bad)).To(MatchError(ContainSubstring("itself an alias"))) }) }) + +var _ = Describe("ModelConfigLoader ResolveAliasName", func() { + var loader *ModelConfigLoader + + BeforeEach(func() { + loader = NewModelConfigLoader("") + loader.configs["real"] = ModelConfig{Name: "real", Backend: "llama-cpp"} + loader.configs["production"] = ModelConfig{Name: "production", Alias: "real"} + loader.configs["chain"] = ModelConfig{Name: "chain", Alias: "production"} + loader.configs["dangling"] = ModelConfig{Name: "dangling", Alias: "nope"} + }) + + It("maps an alias name to the model that actually serves it", func() { + target, isAlias := loader.ResolveAliasName("production") + Expect(isAlias).To(BeTrue()) + Expect(target).To(Equal("real")) + }) + + It("maps a real model name to itself", func() { + target, isAlias := loader.ResolveAliasName("real") + Expect(isAlias).To(BeFalse()) + Expect(target).To(Equal("real")) + }) + + // A rule may be authored for a model that is not installed yet (pre-staging + // placement before standing up a node), so an unknown name must resolve to + // itself rather than to the empty string. + It("maps an unknown name to itself", func() { + target, isAlias := loader.ResolveAliasName("not-installed-yet") + Expect(isAlias).To(BeFalse()) + Expect(target).To(Equal("not-installed-yet")) + }) + + // A broken alias has no model behind it. Resolving to itself keeps the + // caller on a name that simply has no replicas, instead of silently + // governing some other model. + It("maps a dangling alias to itself", func() { + target, isAlias := loader.ResolveAliasName("dangling") + Expect(isAlias).To(BeTrue()) + Expect(target).To(Equal("dangling")) + }) + + It("maps a chained alias to itself rather than following the chain", func() { + target, isAlias := loader.ResolveAliasName("chain") + Expect(isAlias).To(BeTrue()) + Expect(target).To(Equal("chain")) + }) + + It("maps the empty name to itself", func() { + target, isAlias := loader.ResolveAliasName("") + Expect(isAlias).To(BeFalse()) + Expect(target).To(BeEmpty()) + }) +}) diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index bc26baf49..bbae523b1 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -1218,6 +1218,20 @@ func SetSchedulingEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, err.Error())) } + // A rule may be keyed by an alias, in which case it governs whatever + // that alias currently points at. Reject an alias that resolves to + // nothing, and reject a second rule for a model some other rule already + // governs, so the operator hears about the clash instead of silently + // writing a rule that never takes effect. + target, err := registry.ValidateSchedulingTarget(ctx, req.ModelName) + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, nodes.ErrSchedulingConflict) { + status = http.StatusConflict + } + return c.JSON(status, nodeError(status, err.Error())) + } + // Serialize node selector to JSON var selectorJSON string if len(req.NodeSelector) > 0 { @@ -1230,6 +1244,7 @@ func SetSchedulingEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { config := &nodes.ModelSchedulingConfig{ ModelName: req.ModelName, + TargetModel: target, NodeSelector: selectorJSON, MinReplicas: req.MinReplicas, MaxReplicas: req.MaxReplicas, diff --git a/core/http/endpoints/localai/nodes_scheduling_alias_test.go b/core/http/endpoints/localai/nodes_scheduling_alias_test.go new file mode 100644 index 000000000..35065f614 --- /dev/null +++ b/core/http/endpoints/localai/nodes_scheduling_alias_test.go @@ -0,0 +1,125 @@ +package localai + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// aliasResolverStub maps alias names to targets in place of a config loader. +type aliasResolverStub struct{ aliases map[string]string } + +func (s *aliasResolverStub) ResolveAliasName(name string) (string, bool) { + target, ok := s.aliases[name] + if !ok { + return name, false + } + return target, true +} + +var _ = Describe("Scheduling endpoints with model aliases", func() { + var ( + registry *nodes.NodeRegistry + resolver *aliasResolverStub + ) + + BeforeEach(func() { + db := testutil.SetupTestDB() + var err error + registry, err = nodes.NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + resolver = &aliasResolverStub{aliases: map[string]string{"production": "qwen3"}} + registry.SetAliasResolver(resolver) + }) + + post := func(body string) *httptest.ResponseRecorder { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + ExpectWithOffset(1, SetSchedulingEndpoint(registry)(c)).To(Succeed()) + return rec + } + + It("accepts a rule keyed by an alias and reports the model it governs", func() { + rec := post(`{"model_name":"production","min_replicas":2,"node_selector":{"tier":"gpu"}}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var resp map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) + Expect(resp["model_name"]).To(Equal("production")) + Expect(resp["target_model"]).To(Equal("qwen3")) + }) + + It("rejects a second rule for a model an alias rule already governs", func() { + Expect(post(`{"model_name":"production","min_replicas":2}`).Code).To(Equal(http.StatusOK)) + + rec := post(`{"model_name":"qwen3","min_replicas":1}`) + Expect(rec.Code).To(Equal(http.StatusConflict)) + Expect(rec.Body.String()).To(ContainSubstring("production")) + }) + + It("rejects an alias rule for a model that already has its own rule", func() { + Expect(post(`{"model_name":"qwen3","min_replicas":1}`).Code).To(Equal(http.StatusOK)) + + rec := post(`{"model_name":"production","min_replicas":2}`) + Expect(rec.Code).To(Equal(http.StatusConflict)) + Expect(rec.Body.String()).To(ContainSubstring("qwen3")) + }) + + It("still allows editing a rule in place", func() { + Expect(post(`{"model_name":"production","min_replicas":2}`).Code).To(Equal(http.StatusOK)) + + rec := post(`{"model_name":"production","min_replicas":4}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + + stored, err := registry.GetModelScheduling(context.Background(), "production") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.MinReplicas).To(Equal(4)) + }) + + It("rejects a rule keyed by an alias that does not resolve", func() { + resolver.aliases["orphan"] = "orphan" + + rec := post(`{"model_name":"orphan","min_replicas":1}`) + Expect(rec.Code).To(Equal(http.StatusBadRequest)) + Expect(rec.Body.String()).To(ContainSubstring("does not resolve")) + }) + + It("still accepts a rule for a model that is not installed yet", func() { + rec := post(`{"model_name":"not-installed-yet","min_replicas":1}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + }) + + It("labels a rule that another rule shadows when listing", func() { + // A seed file or a repointed alias can leave two rules on one model, + // which the write path above rejects but cannot retract. + Expect(registry.SetModelScheduling(context.Background(), &nodes.ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})).To(Succeed()) + Expect(registry.SetModelScheduling(context.Background(), &nodes.ModelSchedulingConfig{ModelName: "qwen3", MinReplicas: 1})).To(Succeed()) + + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + Expect(ListSchedulingEndpoint(registry)(c)).To(Succeed()) + + var listed []map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &listed)).To(Succeed()) + byName := map[string]map[string]any{} + for _, item := range listed { + byName[item["model_name"].(string)] = item + } + Expect(byName["qwen3"]["shadowed"]).To(BeNil()) + Expect(byName["production"]["shadowed"]).To(Equal(true)) + }) +}) diff --git a/core/http/react-ui/e2e/scheduling.spec.js b/core/http/react-ui/e2e/scheduling.spec.js index 4ce06cb4a..79d781a3f 100644 --- a/core/http/react-ui/e2e/scheduling.spec.js +++ b/core/http/react-ui/e2e/scheduling.spec.js @@ -176,6 +176,75 @@ test.describe('Scheduling page', () => { await expect(page.getByLabel('Node selector').getByText('gpu.vendor=nvidia', { exact: true })).toBeVisible() }) + // A rule may be keyed by an alias, in which case it governs whichever model + // the alias points at. The page has to say which model that is, because the + // rule's own name no longer tells you. + test.describe('rules keyed by a model alias', () => { + const aliasRule = { + model_name: 'production', + target_model: 'llama-3.3', + model_is_alias: true, + node_selector: { tier: 'gpu' }, + min_replicas: 2, + max_replicas: 4, + } + + async function mockAliases(page, aliases = [{ name: 'production', target: 'llama-3.3' }]) { + await page.route('**/api/aliases', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(aliases), + })) + await page.route('**/api/models/capabilities', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ object: 'list', data: [{ id: 'llama-3.3' }, { id: 'production' }] }), + })) + } + + test('names the model an alias rule governs', async ({ page }) => { + await mockScheduling(page, { rules: [aliasRule] }) + await mockAliases(page) + await page.goto('/app/scheduling') + + await expect(page.getByText('production')).toBeVisible() + await expect(page.locator('.scheduling-rule-target')).toHaveText(/llama-3\.3/) + }) + + test('marks a rule another rule already governs as shadowed', async ({ page }) => { + await mockScheduling(page, { rules: [{ ...aliasRule, shadowed: true }, rule] }) + await mockAliases(page) + await page.goto('/app/scheduling') + + await expect(page.locator('.scheduling-rule-shadowed')).toHaveCount(1) + await expect(page.locator('.scheduling-rule-shadowed')).toContainText('Shadowed') + }) + + test('flags an alias rule that no longer resolves', async ({ page }) => { + await mockScheduling(page, { + rules: [{ model_name: 'orphan', target_model: 'orphan', model_is_alias: true, min_replicas: 1 }], + }) + await mockAliases(page, []) + await page.goto('/app/scheduling') + + await expect(page.locator('.scheduling-rule-target--broken')).toBeVisible() + }) + + test('offers aliases in the model picker, tagged with their target', async ({ page }) => { + await mockScheduling(page) + await mockAliases(page) + await page.goto('/app/scheduling') + await page.getByRole('button', { name: 'Add Scheduling Rule' }).click() + + const picker = page.locator('.searchable-model-select input') + await picker.click() + await expect(page.locator('.sms-hint')).toHaveText('alias of llama-3.3') + + await page.getByRole('option', { name: /production/ }).click() + await expect(page.getByText(/production is an alias for llama-3\.3/)).toBeVisible() + }) + }) + test('keeps rule actions reachable on a narrow viewport', async ({ page }) => { await page.setViewportSize({ width: 390, height: 844 }) await mockScheduling(page, { nodeList: nodes.slice(0, 2) }) diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index a5af8fbc6..679e16522 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -2781,6 +2781,31 @@ select.input { justify-content: flex-end; } +/* Second line of a rule's Model cell: the model an alias-keyed rule currently + governs. The cell itself is bold, so the weight is reset here rather than + inherited. */ +.scheduling-rule-target { + font-weight: 400; +} + +.scheduling-rule-target--broken { + font-weight: 400; + color: var(--color-warning); +} + +/* Status pill for a rule another rule already governs, so it has no effect. + Mirrors the unsatisfiable pill's shape. */ +.scheduling-rule-shadowed { + display: inline-block; + font-size: var(--text-xs); + padding: 2px 8px; + border-radius: var(--radius-sm); + font-weight: 600; + background: var(--color-bg-tertiary); + border: 1px solid var(--color-warning); + color: var(--color-warning); +} + @media (max-width: 640px) { .scheduling-rule-actions { width: 100%; diff --git a/core/http/react-ui/src/components/SearchableModelSelect.jsx b/core/http/react-ui/src/components/SearchableModelSelect.jsx index 3d920fa4d..f63902956 100644 --- a/core/http/react-ui/src/components/SearchableModelSelect.jsx +++ b/core/http/react-ui/src/components/SearchableModelSelect.jsx @@ -7,7 +7,10 @@ import { useModels } from '../hooks/useModels' // query isn't treated as a chosen value. After a commit the field is cleared, // matching the add-and-clear flow. Default false keeps the as-you-type // behaviour single-value editors rely on. -export default function SearchableModelSelect({ value, onChange, capability, placeholder = 'Type or select a model...', style, commitOnly = false }) { +// hints: optional { [modelId]: string } shown as muted text beside an entry and +// searchable along with the name. Used to mark aliases with the model they +// point at, so a picker that lists both can tell them apart. +export default function SearchableModelSelect({ value, onChange, capability, placeholder = 'Type or select a model...', style, commitOnly = false, hints = {} }) { const { models, loading } = useModels(capability) const [query, setQuery] = useState('') const [open, setOpen] = useState(false) @@ -29,8 +32,10 @@ export default function SearchableModelSelect({ value, onChange, capability, pla return () => document.removeEventListener('mousedown', handler) }, []) + const needle = query.toLowerCase() const filtered = models.filter(m => - m.id.toLowerCase().includes(query.toLowerCase()) + m.id.toLowerCase().includes(needle) || + (hints[m.id] || '').toLowerCase().includes(needle) ) // Which item Enter will select — matches SearchableSelect behavior @@ -126,6 +131,11 @@ export default function SearchableModelSelect({ value, onChange, capability, pla color: var(--color-primary); font-weight: 600; } + .sms-hint { + color: var(--color-text-muted); + font-size: 0.75rem; + flex-shrink: 0; + } .sms-empty { padding: 8px 10px; font-size: 0.8125rem; @@ -172,6 +182,9 @@ export default function SearchableModelSelect({ value, onChange, capability, pla }} > {m.id} + {hints[m.id] && ( + {hints[m.id]} + )} {isEnterTarget && ( )} diff --git a/core/http/react-ui/src/pages/Scheduling.jsx b/core/http/react-ui/src/pages/Scheduling.jsx index 1558281c3..fc2343ece 100644 --- a/core/http/react-ui/src/pages/Scheduling.jsx +++ b/core/http/react-ui/src/pages/Scheduling.jsx @@ -1,7 +1,7 @@ import { useState, useEffect, useCallback } from 'react' import { useOutletContext } from 'react-router-dom' import { useTranslation } from 'react-i18next' -import { nodesApi } from '../utils/api' +import { nodesApi, modelsApi } from '../utils/api' import PageHeader from '../components/PageHeader' import ConfirmDialog from '../components/ConfirmDialog' import ResponsiveTable from '../components/ResponsiveTable' @@ -66,7 +66,7 @@ function configMode(config) { return 'placement' } -function SchedulingForm({ initialConfig, onSave, onCancel, labels }) { +function SchedulingForm({ initialConfig, onSave, onCancel, labels, aliases }) { const [mode, setMode] = useState(() => configMode(initialConfig)) const [modelName, setModelName] = useState(initialConfig?.model_name || '') // Selector is now a chip-builder map instead of a comma-separated string. @@ -84,6 +84,10 @@ function SchedulingForm({ initialConfig, onSave, onCancel, labels }) { const [minPrefixMatch, setMinPrefixMatch] = useState(initialConfig?.min_prefix_match ?? 0) const hasSelector = Object.keys(selector).length > 0 + // Aliases are listed in the picker alongside models, tagged with the model + // they resolve to so the two are distinguishable in one flat list. + const aliasHints = Object.fromEntries(Object.entries(aliases || {}).map(([name, target]) => [name, `alias of ${target}`])) + const aliasTarget = (aliases || {})[modelName] const isValid = () => { if (!modelName) return false @@ -159,9 +163,20 @@ function SchedulingForm({ initialConfig, onSave, onCancel, labels }) { )} + {/* An alias is a stable name for whichever model currently serves it, + so a rule on one is a rule on a slot rather than on a model. Say + so at the point of choosing, because the consequence (repointing + the alias carries the rule along) is not visible anywhere else. */} + {aliasTarget && ( + +