chore: merge master into distributed transport PR

Keep the version-reporting import from master and omit the unused
sanitize import after the transport changes.

Assisted-by: Codex:GPT-6
This commit is contained in:
localai-org-maint-bot committed 2026-09-28 12:02:15 +00:00
commit 4151ceda85
51 files changed
+1785 -72

No files matched your search

+5
View File
@@ -623,9 +623,13 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
// All dependencies ready — build SmartRouter with all options at once
var conflictResolver nodes.ConcurrencyConflictResolver
var pinnedResolver nodes.PinnedModelResolver
var modelFiles func(string) []string
if configLoader != nil {
conflictResolver = configLoader
pinnedResolver = configLoader
if cfg.SystemState != nil {
modelFiles = declaredModelFiles(configLoader, cfg.SystemState.Model.ModelsPath)
}
}
modelCleanup := nodes.NewModelCleanupService(registry, remoteUnloader)
// Absence is stamped on by distributedSchedulerOptions rather than written
@@ -645,6 +649,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
DataPath: cfg.DataPath,
ConflictResolver: conflictResolver,
PinnedResolver: pinnedResolver,
ModelFiles: modelFiles,
PrefixProvider: prefixProvider,
PrefixConfig: prefixCfg,
Pressure: pressure,
+29
View File
@@ -0,0 +1,29 @@
package application
import (
"path/filepath"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/utils"
)
// declaredModelFiles resolves the files a model needs on disk beyond the ones
// its config names: what its gallery install or import declared under
// `files:`, and what the config itself lists under download_files. The
// distributed router stages these to workers, which cannot see the frontend's
// models directory.
func declaredModelFiles(configLoader *config.ModelConfigLoader, modelsPath string) func(modelName string) []string {
return func(modelName string) []string {
files := gallery.InstalledModelFiles(modelsPath, modelName)
if cfg, ok := configLoader.GetModelConfig(modelName); ok {
for _, f := range cfg.DownloadFiles {
if utils.VerifyPath(f.Filename, modelsPath) != nil {
continue
}
files = append(files, filepath.Join(modelsPath, f.Filename))
}
}
return files
}
}
+41
View File
@@ -0,0 +1,41 @@
package application
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
)
var _ = Describe("declaredModelFiles", func() {
It("combines the gallery install's files with the config's download_files", func() {
modelsPath := GinkgoT().TempDir()
Expect(os.WriteFile(filepath.Join(modelsPath, "big.yaml"), []byte(`
name: big
backend: llama-cpp
parameters:
model: big/Big-00001-of-00002.gguf
download_files:
- filename: big/extra.bin
uri: https://example.com/extra.bin
`), 0o644)).To(Succeed())
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName("big")), []byte(`
files:
- filename: big/Big-00001-of-00002.gguf
- filename: big/Big-00002-of-00002.gguf
`), 0o644)).To(Succeed())
loader := config.NewModelConfigLoader(modelsPath)
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
Expect(declaredModelFiles(loader, modelsPath)("big")).To(ConsistOf(
filepath.Join(modelsPath, "big/Big-00001-of-00002.gguf"),
filepath.Join(modelsPath, "big/Big-00002-of-00002.gguf"),
filepath.Join(modelsPath, "big/extra.bin"),
))
})
})
+2 -2
View File
@@ -88,7 +88,7 @@ func ModelTTS(
// a FS path
mp := filepath.Join(loader.ModelPath, modelConfig.Model)
if _, err := os.Stat(mp); err == nil {
if err := utils.VerifyPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
if err := utils.VerifyResolvedPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
return "", nil, err
}
modelPath = mp
@@ -189,7 +189,7 @@ func ModelTTSStream(
// a FS path
mp := filepath.Join(loader.ModelPath, modelConfig.Model)
if _, err := os.Stat(mp); err == nil {
if err := utils.VerifyPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
if err := utils.VerifyResolvedPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
return err
}
modelPath = mp
+3
View File
@@ -20,6 +20,7 @@ import (
"github.com/mudler/LocalAI/core/services/jobs"
mcpRemote "github.com/mudler/LocalAI/core/services/mcp"
"github.com/mudler/LocalAI/core/services/messaging"
"github.com/mudler/LocalAI/internal"
"github.com/mudler/cogito"
"github.com/mudler/cogito/clients"
"github.com/mudler/xlog"
@@ -163,6 +164,8 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
registrationBody := map[string]any{
"name": nodeName,
"node_type": "agent",
"version": internal.Version,
"commit": internal.Commit,
}
if cmd.RegistrationToken != "" {
registrationBody["token"] = cmd.RegistrationToken
+72
View File
@@ -0,0 +1,72 @@
package gallery_test
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/system"
)
// DeleteModelFromSystem removes files named by a model name and by the
// model's gallery file. Neither may reach outside the models directory: the
// name can come from an API caller or from an assistant tool call, and the
// gallery file is a YAML file on disk.
var _ = Describe("DeleteModelFromSystem path containment", func() {
var (
root string
modelsPath string
outside string
state *system.SystemState
)
BeforeEach(func() {
root = GinkgoT().TempDir()
modelsPath = filepath.Join(root, "models")
outside = filepath.Join(root, "outside")
Expect(os.MkdirAll(modelsPath, 0o755)).To(Succeed())
Expect(os.MkdirAll(outside, 0o755)).To(Succeed())
var err error
state, err = system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).ToNot(HaveOccurred())
})
It("refuses a model name that escapes the models directory", func() {
victim := filepath.Join(outside, "victim.yaml")
Expect(os.WriteFile(victim, []byte("name: victim\n"), 0o644)).To(Succeed())
Expect(gallery.DeleteModelFromSystem(state, "../outside/victim")).ToNot(Succeed())
Expect(victim).To(BeARegularFile())
})
It("does not remove gallery-declared files outside the models directory", func() {
secret := filepath.Join(outside, "secret.bin")
Expect(os.WriteFile(secret, []byte("x"), 0o644)).To(Succeed())
Expect(os.WriteFile(filepath.Join(modelsPath, "m.yaml"), []byte("name: m\n"), 0o644)).To(Succeed())
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName("m")), []byte(`
files:
- filename: ../outside/secret.bin
`), 0o644)).To(Succeed())
_ = gallery.DeleteModelFromSystem(state, "m")
Expect(secret).To(BeARegularFile())
})
It("still deletes a normal model and its declared files", func() {
weights := filepath.Join(modelsPath, "m", "w.gguf")
Expect(os.MkdirAll(filepath.Dir(weights), 0o755)).To(Succeed())
Expect(os.WriteFile(weights, []byte("w"), 0o644)).To(Succeed())
Expect(os.WriteFile(filepath.Join(modelsPath, "m.yaml"), []byte("name: m\nparameters:\n model: m/w.gguf\n"), 0o644)).To(Succeed())
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName("m")), []byte(`
files:
- filename: m/w.gguf
`), 0o644)).To(Succeed())
Expect(gallery.DeleteModelFromSystem(state, "m")).To(Succeed())
Expect(weights).ToNot(BeAnExistingFile())
Expect(filepath.Join(modelsPath, "m.yaml")).ToNot(BeAnExistingFile())
})
})
+46
View File
@@ -0,0 +1,46 @@
package gallery
import (
"os"
"path/filepath"
"strings"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/mudler/xlog"
)
// InstalledModelFiles returns the absolute paths of the files that the install
// of model name declared (the entry's `files:`), as recorded in its gallery
// file. A model config names only the file a backend opens first, while a
// backend can read more by itself (llama.cpp opens the other shards of a split
// GGUF by name), so this is the complete list of what the model needs on disk.
// It returns nil for a model that was not installed from a gallery or import.
func InstalledModelFiles(modelsPath, name string) []string {
// Model names can hold path separators; the gallery file flattens them
// the same way listModelFiles does.
rel := galleryFileName(strings.ReplaceAll(name, string(os.PathSeparator), "__"))
if err := utils.VerifyPath(rel, modelsPath); err != nil {
return nil
}
galleryFile := filepath.Join(modelsPath, rel)
if _, err := os.Stat(galleryFile); err != nil {
return nil
}
cfg, err := ReadConfigFile[ModelConfig](galleryFile)
if err != nil {
xlog.Warn("Failed to read gallery file for installed model files", "model", name, "file", galleryFile, "error", err)
return nil
}
files := make([]string, 0, len(cfg.Files))
for _, f := range cfg.Files {
// VerifyPath joins its argument onto modelsPath itself, so it must
// get the relative name; an absolute path would always pass.
if err := utils.VerifyPath(f.Filename, modelsPath); err != nil {
xlog.Warn("Ignoring declared model file outside the models path", "model", name, "file", f.Filename)
continue
}
files = append(files, filepath.Join(modelsPath, f.Filename))
}
return files
}
+53
View File
@@ -0,0 +1,53 @@
package gallery_test
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/gallery"
)
var _ = Describe("InstalledModelFiles", func() {
var modelsPath string
BeforeEach(func() {
modelsPath = GinkgoT().TempDir()
})
writeGalleryFile := func(name, body string) {
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName(name)), []byte(body), 0o644)).To(Succeed())
}
It("returns the files the install declared, under the models path", func() {
writeGalleryFile("big", `
name: big
files:
- filename: llama-cpp/models/big/Big-00001-of-00002.gguf
uri: huggingface://org/repo/Big-00001-of-00002.gguf
- filename: llama-cpp/models/big/Big-00002-of-00002.gguf
uri: huggingface://org/repo/Big-00002-of-00002.gguf
`)
Expect(gallery.InstalledModelFiles(modelsPath, "big")).To(Equal([]string{
filepath.Join(modelsPath, "llama-cpp/models/big/Big-00001-of-00002.gguf"),
filepath.Join(modelsPath, "llama-cpp/models/big/Big-00002-of-00002.gguf"),
}))
})
It("drops entries that escape the models path", func() {
writeGalleryFile("evil", `
files:
- filename: ../outside.gguf
- filename: ok.gguf
`)
Expect(gallery.InstalledModelFiles(modelsPath, "evil")).To(Equal([]string{
filepath.Join(modelsPath, "ok.gguf"),
}))
})
It("returns nothing for a model that was not installed from a gallery", func() {
Expect(gallery.InstalledModelFiles(modelsPath, "handwritten")).To(BeEmpty())
})
})
+7 -4
View File
@@ -808,8 +808,11 @@ func GetLocalModelConfiguration(basePath string, name string) (*ModelConfig, err
func listModelFiles(systemState *system.SystemState, name string) ([]string, error) {
// VerifyPath joins its argument onto the models path itself, so every
// check below passes the relative name: an already-joined absolute path
// always lands inside the base and the check would pass anything.
configFile := filepath.Join(systemState.Model.ModelsPath, fmt.Sprintf("%s.yaml", name))
if err := utils.VerifyPath(configFile, systemState.Model.ModelsPath); err != nil {
if err := utils.VerifyPath(fmt.Sprintf("%s.yaml", name), systemState.Model.ModelsPath); err != nil {
return nil, fmt.Errorf("failed to verify path %s: %w", configFile, err)
}
@@ -817,7 +820,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
galleryFile := filepath.Join(systemState.Model.ModelsPath, galleryFileName(name))
if err := utils.VerifyPath(galleryFile, systemState.Model.ModelsPath); err != nil {
if err := utils.VerifyPath(galleryFileName(name), systemState.Model.ModelsPath); err != nil {
return nil, fmt.Errorf("failed to verify path %s: %w", galleryFile, err)
}
@@ -847,7 +850,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
if err == nil && galleryconfig != nil {
for _, f := range galleryconfig.Files {
fullPath := filepath.Join(systemState.Model.ModelsPath, f.Filename)
if err := utils.VerifyPath(fullPath, systemState.Model.ModelsPath); err != nil {
if err := utils.VerifyPath(f.Filename, systemState.Model.ModelsPath); err != nil {
return allFiles, fmt.Errorf("failed to verify path %s: %w", fullPath, err)
}
allFiles = append(allFiles, fullPath)
@@ -858,7 +861,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
for _, f := range additionalFiles {
fullPath := filepath.Join(filepath.Join(systemState.Model.ModelsPath, f))
if err := utils.VerifyPath(fullPath, systemState.Model.ModelsPath); err != nil {
if err := utils.VerifyPath(f, systemState.Model.ModelsPath); err != nil {
return allFiles, fmt.Errorf("failed to verify path %s: %w", fullPath, err)
}
allFiles = append(allFiles, fullPath)
+7
View File
@@ -114,6 +114,11 @@ type RegisterNodeRequest struct {
// VRAMBudget is the worker's operator-set VRAM cap ("80%" or "12GB"). The
// registry resolves and enforces it against the raw reported VRAM.
VRAMBudget string `json:"vram_budget,omitempty"`
// Version is the LocalAI build version reported by the worker at
// registration. Empty for workers registered before this field existed.
Version string `json:"version,omitempty"`
// Commit is the git commit hash the worker binary was built from.
Commit string `json:"commit,omitempty"`
}
// RegisterNodeEndpoint registers a new backend node.
@@ -191,6 +196,8 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au
Capability: req.Capability,
MaxReplicasPerModel: maxReplicasPerModel,
VRAMBudget: req.VRAMBudget,
Version: req.Version,
Commit: req.Commit,
}
ctx := c.Request().Context()
+7
View File
@@ -9966,6 +9966,13 @@ button.collapsible-header:focus-visible {
.node-inspector__actions .btn { justify-content: center; min-width: 0; }
.node-inspector__back { align-items: center; background: transparent; border: 0; color: var(--color-primary); cursor: pointer; display: flex; font: inherit; font-size: var(--text-xs); gap: 6px; max-width: 285px; overflow: hidden; padding: 3px 0; text-overflow: ellipsis; white-space: nowrap; }
.node-inspector__back:focus-visible { border-radius: var(--radius-sm); outline: 2px solid var(--color-primary); outline-offset: 3px; }
.node-inspector__models { margin: 10px 0 0; }
.node-inspector__models > dd { margin: 0; }
.node-inspector__model-list { display: grid; gap: 4px; list-style: none; margin: 6px 0 0; padding: 0; }
.node-inspector__model-row { align-items: center; display: flex; flex-wrap: wrap; gap: 6px; font-size: var(--text-xs); }
.node-inspector__model-row .cell-mono { font-family: var(--font-mono); font-size: .625rem; overflow-wrap: anywhere; }
.node-inspector__model-row .state-pill { border-radius: var(--radius-full); font-size: .5625rem; font-weight: 600; padding: 1px 7px; text-transform: capitalize; }
.node-inspector__model-row .text-muted { font-size: .5625rem; }
.model-inspector__backends { margin-top: 10px; }
.model-inspector__nodes { display: grid; gap: 9px; }
.model-inspector__node { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-md); padding: 10px; }
@@ -1,6 +1,6 @@
import { useEffect, useRef, useState } from 'react'
import StatusPill from './StatusPill'
import { formatBytes, formatCapacity, timeAgo } from './nodeStatus'
import { formatBytes, formatCapacity, timeAgo, modelStateConfig } from './nodeStatus'
import { nodesApi } from '../../utils/api'
import { capacityReading, nodeLifecycleAction } from '../../utils/nodeFleet'
import useInspectorDrawer from './useInspectorDrawer'
@@ -24,6 +24,8 @@ function ResourceBar({ label, total, available, tone }) {
export default function NodeInspector({ node, open, onClose, onApprove, onDrain, onResume, onBack, backLabel }) {
const [backends, setBackends] = useState(null)
const [backendError, setBackendError] = useState('')
const [models, setModels] = useState(null)
const [modelError, setModelError] = useState('')
const nodeId = node?.id
const backRef = useRef(null)
const closeRef = useRef(null)
@@ -48,6 +50,19 @@ export default function NodeInspector({ node, open, onClose, onApprove, onDrain,
return () => { current = false }
}, [open, nodeId])
useEffect(() => {
if (!open || !nodeId) return undefined
let current = true
setModels(null)
setModelError('')
nodesApi.getModels(nodeId).then(data => {
if (current) setModels(Array.isArray(data) ? data : [])
}).catch(error => {
if (current) setModelError(error.message || 'Unable to load models')
})
return () => { current = false }
}, [open, nodeId])
if (!open || !node) return null
const cpuKnown = node.cpu_logical_cores > 0 && Number.isFinite(node.cpu_usage_percent) && Number.isFinite(node.cpu_load_1)
const disk = capacityReading(node.total_disk, node.available_disk)
@@ -74,6 +89,7 @@ export default function NodeInspector({ node, open, onClose, onApprove, onDrain,
<h3>Node</h3>
<dl className="node-inspector__metrics">
<InspectorMetric label="Address"><span className="node-inspector__address">{node.address || 'No address reported'}</span></InspectorMetric>
<InspectorMetric label="Version">{node.version || '—'}</InspectorMetric>
<InspectorMetric label="Heartbeat">{timeAgo(node.last_heartbeat)}</InspectorMetric>
</dl>
<div className="node-inspector__labels" aria-label="Node labels">{Object.keys(node.labels || {}).length ? Object.entries(node.labels).map(([key, value]) => <span key={key}>{key}={value}</span>) : <span className="text-muted">No labels</span>}</div>
@@ -94,6 +110,26 @@ export default function NodeInspector({ node, open, onClose, onApprove, onDrain,
<InspectorMetric label="Backends">{backendError ? <span className="text-error">{backendError}</span> : backends === null ? 'Loading…' : `${backends.length} backend${backends.length === 1 ? '' : 's'}`}</InspectorMetric>
<InspectorMetric label="In-flight work">{node.in_flight_count ?? 0}</InspectorMetric>
</dl>
<div className="node-inspector__models">
<dt className="drawer-eyebrow">Running models</dt>
<dd>
{modelError ? <span className="text-error">{modelError}</span>
: models === null ? <span className="text-muted">Loading…</span>
: models.length === 0 ? <span className="text-muted">No models loaded</span>
: <ul className="node-inspector__model-list">
{models.map(model => {
const stCfg = modelStateConfig[model.state] || modelStateConfig.idle
return (
<li key={model.id || `${model.model_name}#${model.replica_index}`} className="node-inspector__model-row">
<span className="cell-mono">{model.model_name}</span>
<span className="state-pill" style={{ background: stCfg.bg, color: stCfg.color, border: `1px solid ${stCfg.border}` }}>{model.state}</span>
<span className="text-muted">{model.in_flight ?? 0} in flight</span>
</li>
)
})}
</ul>}
</dd>
</div>
</section>
</div>
<footer className="node-inspector__actions">
@@ -138,6 +138,10 @@ export default function NodeDetail() {
<div className="drawer-eyebrow">In-flight</div>
<span className="cell-mono">{node.in_flight_count || 0}</span>
</div>
<div>
<div className="drawer-eyebrow">Version</div>
<span className="cell-mono">{node.version || '—'}</span>
</div>
<div>
<div className="drawer-eyebrow">Heartbeat</div>
<span>{timeAgo(node.last_heartbeat)}</span>
+4 -4
View File
@@ -93,7 +93,7 @@ func (s *ConfigService) GetConfig(_ context.Context, name string) (*ConfigView,
if configPath == "" {
return nil, ErrConfigFileMissing
}
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
}
data, err := os.ReadFile(configPath)
@@ -137,7 +137,7 @@ func (s *ConfigService) patchConfig(ctx context.Context, name string, patch map[
return nil, fmt.Errorf("%w: PATCH cannot rename model %q to %q; use the model edit endpoint", ErrInvalidConfig, name, patchedName)
}
configPath := cfg.GetModelConfigFile()
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
}
diskYAML, err := os.ReadFile(configPath)
@@ -289,7 +289,7 @@ func (s *ConfigService) editYAML(ctx context.Context, name string, body []byte)
configPath := existing.GetModelConfigFile()
modelsPath := s.modelsPath()
if err := utils.VerifyPath(configPath, modelsPath); err != nil {
if err := utils.VerifyResolvedPath(configPath, modelsPath); err != nil {
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
}
@@ -304,7 +304,7 @@ func (s *ConfigService) editYAML(ctx context.Context, name string, body []byte)
}
newConfigPath := filepath.Join(modelsPath, req.Name+".yaml")
paths = append(paths, newConfigPath, filepath.Join(modelsPath, gallery.GalleryFileName(name)), filepath.Join(modelsPath, gallery.GalleryFileName(req.Name)))
if err := utils.VerifyPath(newConfigPath, modelsPath); err != nil {
if err := utils.VerifyPath(req.Name+".yaml", modelsPath); err != nil {
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
}
if _, err := os.Stat(newConfigPath); err == nil {
@@ -0,0 +1,58 @@
package modeladmin
import (
"context"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// A model config can be loaded from outside the models directory (for
// example with --config-file). The admin mutations write the config file
// back, so they must refuse a file outside the models directory rather than
// write wherever the loader found it.
var _ = Describe("ConfigService config file containment", func() {
var (
svc *ConfigService
ctx context.Context
outside string
orig []byte
)
BeforeEach(func() {
svc, _ = newTestService()
ctx = context.Background()
outside = filepath.Join(GinkgoT().TempDir(), "external.yaml")
orig = []byte("name: external\nbackend: llama-cpp\n")
Expect(os.WriteFile(outside, orig, 0o644)).To(Succeed())
Expect(svc.Loader.ReadModelConfig(outside, svc.AppConfig.ToConfigLoaderOptions()...)).To(Succeed())
cfg, ok := svc.Loader.GetModelConfig("external")
Expect(ok).To(BeTrue())
Expect(cfg.GetModelConfigFile()).To(Equal(outside))
})
It("refuses to pin a model whose config file is outside the models directory", func() {
_, err := svc.TogglePinned(ctx, "external", ActionPin, nil)
Expect(err).To(MatchError(ErrPathNotTrusted))
Expect(os.ReadFile(outside)).To(Equal(orig))
})
It("refuses to toggle the state of such a model", func() {
_, err := svc.ToggleState(ctx, "external", ActionDisable)
Expect(err).To(MatchError(ErrPathNotTrusted))
Expect(os.ReadFile(outside)).To(Equal(orig))
})
It("refuses to patch such a model", func() {
_, err := svc.PatchConfig(ctx, "external", map[string]any{"context_size": 4096})
Expect(err).To(MatchError(ErrPathNotTrusted))
Expect(os.ReadFile(outside)).To(Equal(orig))
})
It("refuses to read such a model's config", func() {
_, err := svc.GetConfig(ctx, "external")
Expect(err).To(MatchError(ErrPathNotTrusted))
})
})
+1 -1
View File
@@ -29,7 +29,7 @@ func (s *ConfigService) TogglePinned(_ context.Context, name string, action Acti
if configPath == "" {
return nil, ErrConfigFileMissing
}
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
}
if err := mutateYAMLBoolFlag(configPath, "pinned", action == ActionPin); err != nil {
+1 -1
View File
@@ -49,7 +49,7 @@ func (s *ConfigService) toggleState(ctx context.Context, name string, action Act
if configPath == "" {
return nil, ErrConfigFileMissing
}
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
}
var result *ToggleResult
+78
View File
@@ -0,0 +1,78 @@
package nodes
import (
"os"
"path/filepath"
"strings"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/xlog"
)
// declaredExtraFiles returns the files the model's install declared that the
// path fields of opts do not already stage: neither named by a field nor
// inside a directory a field names. It must run on the local paths, before
// staging rewrites the fields to remote ones.
func (r *SmartRouter) declaredExtraFiles(trackingKey string, opts *pb.ModelOptions) []string {
if r.modelFiles == nil || opts == nil || trackingKey == "" {
return nil
}
covered := append([]string{
opts.ModelFile, opts.MMProj, opts.LoraAdapter, opts.DraftModel,
opts.CLIPModel, opts.Tokenizer, opts.AudioPath, opts.LoraBase,
}, opts.LoraAdapters...)
seen := map[string]struct{}{}
var extra []string
for _, p := range r.modelFiles(trackingKey) {
p = filepath.Clean(p)
if _, dup := seen[p]; dup || coveredByField(p, covered) {
continue
}
seen[p] = struct{}{}
extra = append(extra, p)
}
return extra
}
func coveredByField(path string, fields []string) bool {
for _, f := range fields {
if f == "" {
continue
}
f = filepath.Clean(f)
if path == f || strings.HasPrefix(path, f+string(filepath.Separator)) {
return true
}
}
return false
}
// existingFiles drops declared files that are not on the frontend. An install
// can declare files that are gone by load time (an archive unpacked and then
// removed, say), so a missing one is not a reason to refuse the load; the
// backend reports it if it really needed it.
func existingFiles(paths []string, nodeName, trackingKey string) []string {
out := paths[:0:0]
for _, p := range paths {
if _, err := os.Stat(p); err != nil {
xlog.Warn("Skipping staging for declared model file that is not on the frontend", "path", p, "node", nodeName, "model", trackingKey, "error", err)
continue
}
out = append(out, p)
}
return out
}
// stagingPayloadBytes totals the on-disk size of everything staging uploads
// for a model: the path fields plus the declared files they do not cover. The
// first shard of a split GGUF can be a few MB of metadata while the weights
// sit in the others, so sizing the fields alone starves the load budget and
// the disk-headroom check.
func (r *SmartRouter) stagingPayloadBytes(trackingKey string, opts *pb.ModelOptions) int64 {
total := modelPayloadBytes(opts)
for _, p := range r.declaredExtraFiles(trackingKey, opts) {
total += pathBytes(p)
}
return total
}
+5
View File
@@ -124,6 +124,11 @@ type BackendNode struct {
// worker's re-registration value does not clobber it (mirrors
// MaxReplicasPerModelManuallySet).
VRAMBudgetManuallySet bool `gorm:"column:vram_budget_manually_set;default:false" json:"vram_budget_manually_set"`
// Version is the LocalAI build version reported by the worker at
// registration. Empty for workers registered before this field existed.
Version string `gorm:"column:version;size:64" json:"version,omitempty"`
// Commit is the git commit hash the worker binary was built from.
Commit string `gorm:"column:commit;size:64" json:"commit,omitempty"`
APIKeyID string `gorm:"size:36" json:"-"` // auto-provisioned API key ID (for cleanup)
AuthUserID string `gorm:"size:36" json:"-"` // auto-provisioned user ID (for cleanup)
LastHeartbeat time.Time `gorm:"column:last_heartbeat" json:"last_heartbeat"`
+32 -2
View File
@@ -73,6 +73,12 @@ type SmartRouterOptions struct {
// nil disables the exclusion. Deliberate teardown (UnloadModel, admin
// endpoints, node drain) is unaffected.
PinnedResolver PinnedModelResolver
// ModelFiles, when set, returns the absolute local paths of every file a
// model's install declared (gallery `files:`, config `download_files`).
// The path fields of a load request name only what the backend opens
// first; this is how staging learns about the rest, such as the other
// shards of a split GGUF. nil stages the path fields alone.
ModelFiles func(modelName string) []string
// PrefixProvider, when set, enables prefix-cache-aware routing: requests
// carrying a prompt prefix chain (distributedhdr.PrefixChain) are biased
// toward the node that already holds the longest matching prefix, subject
@@ -189,6 +195,9 @@ type SmartRouter struct {
// pinnedResolver feeds the eviction paths the set of pinned model names
// (see SmartRouterOptions.PinnedResolver). nil disables the exclusion.
pinnedResolver PinnedModelResolver
// modelFiles resolves a model's declared files (see
// SmartRouterOptions.ModelFiles). nil stages the path fields alone.
modelFiles func(modelName string) []string
// prefixProvider is the prefix-cache routing seam (nil disables it; see
// SmartRouterOptions.PrefixProvider). prefixConfig holds the global policy
// and thresholds.
@@ -283,6 +292,7 @@ func NewSmartRouter(registry ModelRouter, opts SmartRouterOptions) *SmartRouter
stagingTracker: NewStagingTracker(),
conflictResolver: opts.ConflictResolver,
pinnedResolver: opts.PinnedResolver,
modelFiles: opts.ModelFiles,
probeCache: newProbeCache(probeCacheTTL),
prefixProvider: opts.PrefixProvider,
prefixConfig: opts.PrefixConfig,
@@ -425,7 +435,7 @@ func (r *SmartRouter) scheduleAndLoad(ctx context.Context, backendType, tracking
// Size the remote load budget BEFORE staging: stageModelFiles rewrites the
// path fields to their remote equivalents on a clone, and only the local
// paths can be stat'ed here.
payloadBytes := modelPayloadBytes(modelOpts)
payloadBytes := r.stagingPayloadBytes(trackingKey, modelOpts)
loadTimeout := r.loadTimeoutFor(payloadBytes)
// Pre-stage model files via FileStager before loading
@@ -1367,7 +1377,7 @@ func (r *SmartRouter) narrowByDiskHeadroom(ctx context.Context, modelID string,
return candidateNodeIDs, nil
}
requiredDisk := DiskRequirementFor(modelPayloadBytes(modelOpts))
requiredDisk := DiskRequirementFor(r.stagingPayloadBytes(modelID, modelOpts))
diskCandidates, diskErr := r.registry.NarrowByDiskHeadroom(ctx, candidateNodeIDs, requiredDisk)
// The check runs even when disabled. "Disabled" means do not BLOCK, not do
@@ -1568,6 +1578,10 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
localModelDir = filepath.Dir(opts.ModelFile)
}
// Resolved before the path fields are rewritten to remote paths below,
// since that is what tells which declared files the fields already cover.
declared := existingFiles(r.declaredExtraFiles(trackingKey, opts), node.Name, trackingKey)
// keyMapper generates storage keys namespaced under trackingKey, preserving
// subdirectory structure relative to frontendModelsDir. This ensures:
// 1. All files for a model land in one directory on the worker for clean deletion
@@ -1614,6 +1628,7 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
totalFiles++
}
}
totalFiles += len(declared)
// Start tracking staging progress
r.stagingTracker.Start(trackingKey, node.Name, totalFiles)
@@ -1757,6 +1772,21 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
}
}
for _, localPath := range declared {
fileIdx++
fileName := filepath.Base(localPath)
stageCtx := r.withStagingCallback(ctx, trackingKey, fileName, fileIdx, totalFiles)
xlog.Info("Staging declared model file", "model", trackingKey, "node", node.Name, "file", fileName, "fileIndex", fileIdx, "totalFiles", totalFiles)
if _, err := r.fileStager.EnsureRemote(stageCtx, node.ID, localPath, keyMapper.Key(localPath)); err != nil {
// The install declared it, so the backend may read it: loading
// without it fails later with a less useful error.
xlog.Error("Failed to stage declared model file for remote node", "node", node.Name, "path", localPath, "error", err)
return nil, fmt.Errorf("staging declared model file %s: %w", localPath, err)
}
r.stagingTracker.FileComplete(trackingKey, fileIdx, totalFiles)
}
// Stage file paths referenced in generic Options (key:value pairs where values
// are file paths). Options stay as relative paths — backends resolve them via ModelPath.
for _, options := range [][]string{opts.Options, opts.Overrides} {
@@ -0,0 +1,126 @@
package nodes
import (
"context"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
)
// A model's config names only the file the backend opens first, but its
// install can declare more that the backend reads by itself: llama.cpp opens
// the "-0000N-of-0000M" shards of a split GGUF from the directory of the first
// one. The worker has no view of the frontend's models directory, so every
// declared file must be staged, or the load fails with "failed to load GGUF
// split".
var _ = Describe("stageModelFiles declared model files", func() {
var (
stager *fakeFileStager
router *SmartRouter
node *BackendNode
modelDir string
shards []string
mmproj string
declared map[string][]string
)
BeforeEach(func() {
stager = &fakeFileStager{}
declared = map[string][]string{}
router = &SmartRouter{
fileStager: stager,
stagingTracker: NewStagingTracker(),
modelFiles: func(name string) []string { return declared[name] },
}
node = &BackendNode{ID: "node-1", Name: "node-1", Address: "10.0.0.1:50051"}
root := GinkgoT().TempDir()
modelDir = filepath.Join(root, "llama-cpp", "models", "big")
Expect(os.MkdirAll(modelDir, 0o755)).To(Succeed())
shards = nil
for _, name := range []string{
"Big-Q4_K_M-00001-of-00003.gguf",
"Big-Q4_K_M-00002-of-00003.gguf",
"Big-Q4_K_M-00003-of-00003.gguf",
} {
p := filepath.Join(modelDir, name)
Expect(os.WriteFile(p, []byte("shard "+name), 0o644)).To(Succeed())
shards = append(shards, p)
}
mmproj = filepath.Join(root, "llama-cpp", "mmproj", "big", "mmproj.gguf")
Expect(os.MkdirAll(filepath.Dir(mmproj), 0o755)).To(Succeed())
Expect(os.WriteFile(mmproj, []byte("mmproj"), 0o644)).To(Succeed())
})
opts := func() *pb.ModelOptions {
return &pb.ModelOptions{
Model: "llama-cpp/models/big/Big-Q4_K_M-00001-of-00003.gguf",
ModelFile: shards[0],
MMProj: mmproj,
}
}
stagedPaths := func() []string {
out := make([]string, 0, len(stager.ensureCalls))
for _, c := range stager.ensureCalls {
out = append(out, c.localPath)
}
return out
}
It("stages every declared file once, beside the ones the config names", func() {
declared["big"] = append(append([]string{}, shards...), mmproj)
staged, err := router.stageModelFiles(context.Background(), node, opts(), "big")
Expect(err).ToNot(HaveOccurred())
Expect(stagedPaths()).To(ConsistOf(shards[0], mmproj, shards[1], shards[2]))
// llama.cpp derives the other shards' paths from the first one, so
// they must land in the same remote directory.
for _, c := range stager.ensureCalls {
if c.localPath != mmproj {
Expect(filepath.Dir(c.key)).To(Equal(filepath.Dir(stager.ensureCalls[0].key)))
}
}
Expect(staged.ModelFile).To(Equal("/remote/" + stager.ensureCalls[0].key))
})
It("sizes declared files for the load budget and disk check", func() {
declared["big"] = append(append([]string{}, shards...), mmproj)
var want int64
for _, p := range append(append([]string{}, shards...), mmproj) {
fi, err := os.Stat(p)
Expect(err).ToNot(HaveOccurred())
want += fi.Size()
}
Expect(router.stagingPayloadBytes("big", opts())).To(Equal(want))
})
It("skips a declared file that is missing locally instead of failing", func() {
declared["big"] = append(append([]string{}, shards...), filepath.Join(modelDir, "gone.bin"))
_, err := router.stageModelFiles(context.Background(), node, opts(), "big")
Expect(err).ToNot(HaveOccurred())
Expect(stagedPaths()).To(ConsistOf(shards[0], mmproj, shards[1], shards[2]))
})
It("does not stage a declared file twice when a directory field covers it", func() {
declared["dir"] = []string{shards[1]}
_, err := router.stageModelFiles(context.Background(), node,
&pb.ModelOptions{Model: "llama-cpp/models/big", ModelFile: modelDir}, "dir")
Expect(err).ToNot(HaveOccurred())
Expect(stagedPaths()).To(ConsistOf(shards[0], shards[1], shards[2]))
})
It("stages only the named files for a model that declares none", func() {
_, err := router.stageModelFiles(context.Background(), node, opts(), "handwritten")
Expect(err).ToNot(HaveOccurred())
Expect(stagedPaths()).To(ConsistOf(shards[0], mmproj))
})
})
+3
View File
@@ -8,6 +8,7 @@ import (
"strconv"
"strings"
"github.com/mudler/LocalAI/internal"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/mudler/xlog"
@@ -154,6 +155,8 @@ func (cfg *Config) registrationBody() map[string]any {
"gpu_compute_capability": gpuComputeCap,
"capability": capability,
"max_replicas_per_model": maxReplicas,
"version": internal.Version,
"commit": internal.Commit,
}
// Report free space on the filesystem that backs the MODELS directory.
+2 -2
View File
@@ -41,7 +41,7 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal
// Check if it's a model gallery, or print a warning
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity, installOptions...)
if e != nil && found {
xlog.Error("[startup] failed installing model", "error", err, "model", url)
xlog.Error("[startup] failed installing model", "error", e, "model", url)
err = errors.Join(err, e)
} else if !found {
xlog.Debug("[startup] model not found in the gallery", "model", url)
@@ -54,7 +54,7 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal
modelConfig, discoverErr := importers.DiscoverModelConfig(url, json.RawMessage{})
if discoverErr != nil {
xlog.Error("[startup] failed to discover model config", "error", discoverErr, "model", url)
err = errors.Join(discoverErr, fmt.Errorf("failed to discover model config: %w", err))
err = errors.Join(err, fmt.Errorf("failed to discover model config: %w", discoverErr))
continue
}