mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-30 01:54:31 -04:00
chore: merge master into distributed transport PR
Keep the version-reporting import from master and omit the unused sanitize import after the transport changes. Assisted-by: Codex:GPT-6
This commit is contained in:
commit
4151ceda85
51 files changed
+1785
-72
No files matched your search
@@ -623,9 +623,13 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
|
||||
// All dependencies ready — build SmartRouter with all options at once
|
||||
var conflictResolver nodes.ConcurrencyConflictResolver
|
||||
var pinnedResolver nodes.PinnedModelResolver
|
||||
var modelFiles func(string) []string
|
||||
if configLoader != nil {
|
||||
conflictResolver = configLoader
|
||||
pinnedResolver = configLoader
|
||||
if cfg.SystemState != nil {
|
||||
modelFiles = declaredModelFiles(configLoader, cfg.SystemState.Model.ModelsPath)
|
||||
}
|
||||
}
|
||||
modelCleanup := nodes.NewModelCleanupService(registry, remoteUnloader)
|
||||
// Absence is stamped on by distributedSchedulerOptions rather than written
|
||||
@@ -645,6 +649,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
|
||||
DataPath: cfg.DataPath,
|
||||
ConflictResolver: conflictResolver,
|
||||
PinnedResolver: pinnedResolver,
|
||||
ModelFiles: modelFiles,
|
||||
PrefixProvider: prefixProvider,
|
||||
PrefixConfig: prefixCfg,
|
||||
Pressure: pressure,
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/pkg/utils"
|
||||
)
|
||||
|
||||
// declaredModelFiles resolves the files a model needs on disk beyond the ones
|
||||
// its config names: what its gallery install or import declared under
|
||||
// `files:`, and what the config itself lists under download_files. The
|
||||
// distributed router stages these to workers, which cannot see the frontend's
|
||||
// models directory.
|
||||
func declaredModelFiles(configLoader *config.ModelConfigLoader, modelsPath string) func(modelName string) []string {
|
||||
return func(modelName string) []string {
|
||||
files := gallery.InstalledModelFiles(modelsPath, modelName)
|
||||
if cfg, ok := configLoader.GetModelConfig(modelName); ok {
|
||||
for _, f := range cfg.DownloadFiles {
|
||||
if utils.VerifyPath(f.Filename, modelsPath) != nil {
|
||||
continue
|
||||
}
|
||||
files = append(files, filepath.Join(modelsPath, f.Filename))
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
)
|
||||
|
||||
var _ = Describe("declaredModelFiles", func() {
|
||||
It("combines the gallery install's files with the config's download_files", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, "big.yaml"), []byte(`
|
||||
name: big
|
||||
backend: llama-cpp
|
||||
parameters:
|
||||
model: big/Big-00001-of-00002.gguf
|
||||
download_files:
|
||||
- filename: big/extra.bin
|
||||
uri: https://example.com/extra.bin
|
||||
`), 0o644)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName("big")), []byte(`
|
||||
files:
|
||||
- filename: big/Big-00001-of-00002.gguf
|
||||
- filename: big/Big-00002-of-00002.gguf
|
||||
`), 0o644)).To(Succeed())
|
||||
|
||||
loader := config.NewModelConfigLoader(modelsPath)
|
||||
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
|
||||
|
||||
Expect(declaredModelFiles(loader, modelsPath)("big")).To(ConsistOf(
|
||||
filepath.Join(modelsPath, "big/Big-00001-of-00002.gguf"),
|
||||
filepath.Join(modelsPath, "big/Big-00002-of-00002.gguf"),
|
||||
filepath.Join(modelsPath, "big/extra.bin"),
|
||||
))
|
||||
})
|
||||
})
|
||||
+2
-2
@@ -88,7 +88,7 @@ func ModelTTS(
|
||||
// a FS path
|
||||
mp := filepath.Join(loader.ModelPath, modelConfig.Model)
|
||||
if _, err := os.Stat(mp); err == nil {
|
||||
if err := utils.VerifyPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
|
||||
if err := utils.VerifyResolvedPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
modelPath = mp
|
||||
@@ -189,7 +189,7 @@ func ModelTTSStream(
|
||||
// a FS path
|
||||
mp := filepath.Join(loader.ModelPath, modelConfig.Model)
|
||||
if _, err := os.Stat(mp); err == nil {
|
||||
if err := utils.VerifyPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
|
||||
if err := utils.VerifyResolvedPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
|
||||
return err
|
||||
}
|
||||
modelPath = mp
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/services/jobs"
|
||||
mcpRemote "github.com/mudler/LocalAI/core/services/mcp"
|
||||
"github.com/mudler/LocalAI/core/services/messaging"
|
||||
"github.com/mudler/LocalAI/internal"
|
||||
"github.com/mudler/cogito"
|
||||
"github.com/mudler/cogito/clients"
|
||||
"github.com/mudler/xlog"
|
||||
@@ -163,6 +164,8 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
|
||||
registrationBody := map[string]any{
|
||||
"name": nodeName,
|
||||
"node_type": "agent",
|
||||
"version": internal.Version,
|
||||
"commit": internal.Commit,
|
||||
}
|
||||
if cmd.RegistrationToken != "" {
|
||||
registrationBody["token"] = cmd.RegistrationToken
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package gallery_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
)
|
||||
|
||||
// DeleteModelFromSystem removes files named by a model name and by the
|
||||
// model's gallery file. Neither may reach outside the models directory: the
|
||||
// name can come from an API caller or from an assistant tool call, and the
|
||||
// gallery file is a YAML file on disk.
|
||||
var _ = Describe("DeleteModelFromSystem path containment", func() {
|
||||
var (
|
||||
root string
|
||||
modelsPath string
|
||||
outside string
|
||||
state *system.SystemState
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
root = GinkgoT().TempDir()
|
||||
modelsPath = filepath.Join(root, "models")
|
||||
outside = filepath.Join(root, "outside")
|
||||
Expect(os.MkdirAll(modelsPath, 0o755)).To(Succeed())
|
||||
Expect(os.MkdirAll(outside, 0o755)).To(Succeed())
|
||||
var err error
|
||||
state, err = system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("refuses a model name that escapes the models directory", func() {
|
||||
victim := filepath.Join(outside, "victim.yaml")
|
||||
Expect(os.WriteFile(victim, []byte("name: victim\n"), 0o644)).To(Succeed())
|
||||
|
||||
Expect(gallery.DeleteModelFromSystem(state, "../outside/victim")).ToNot(Succeed())
|
||||
Expect(victim).To(BeARegularFile())
|
||||
})
|
||||
|
||||
It("does not remove gallery-declared files outside the models directory", func() {
|
||||
secret := filepath.Join(outside, "secret.bin")
|
||||
Expect(os.WriteFile(secret, []byte("x"), 0o644)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, "m.yaml"), []byte("name: m\n"), 0o644)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName("m")), []byte(`
|
||||
files:
|
||||
- filename: ../outside/secret.bin
|
||||
`), 0o644)).To(Succeed())
|
||||
|
||||
_ = gallery.DeleteModelFromSystem(state, "m")
|
||||
Expect(secret).To(BeARegularFile())
|
||||
})
|
||||
|
||||
It("still deletes a normal model and its declared files", func() {
|
||||
weights := filepath.Join(modelsPath, "m", "w.gguf")
|
||||
Expect(os.MkdirAll(filepath.Dir(weights), 0o755)).To(Succeed())
|
||||
Expect(os.WriteFile(weights, []byte("w"), 0o644)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, "m.yaml"), []byte("name: m\nparameters:\n model: m/w.gguf\n"), 0o644)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName("m")), []byte(`
|
||||
files:
|
||||
- filename: m/w.gguf
|
||||
`), 0o644)).To(Succeed())
|
||||
|
||||
Expect(gallery.DeleteModelFromSystem(state, "m")).To(Succeed())
|
||||
Expect(weights).ToNot(BeAnExistingFile())
|
||||
Expect(filepath.Join(modelsPath, "m.yaml")).ToNot(BeAnExistingFile())
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,46 @@
|
||||
package gallery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/utils"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
|
||||
// InstalledModelFiles returns the absolute paths of the files that the install
|
||||
// of model name declared (the entry's `files:`), as recorded in its gallery
|
||||
// file. A model config names only the file a backend opens first, while a
|
||||
// backend can read more by itself (llama.cpp opens the other shards of a split
|
||||
// GGUF by name), so this is the complete list of what the model needs on disk.
|
||||
// It returns nil for a model that was not installed from a gallery or import.
|
||||
func InstalledModelFiles(modelsPath, name string) []string {
|
||||
// Model names can hold path separators; the gallery file flattens them
|
||||
// the same way listModelFiles does.
|
||||
rel := galleryFileName(strings.ReplaceAll(name, string(os.PathSeparator), "__"))
|
||||
if err := utils.VerifyPath(rel, modelsPath); err != nil {
|
||||
return nil
|
||||
}
|
||||
galleryFile := filepath.Join(modelsPath, rel)
|
||||
if _, err := os.Stat(galleryFile); err != nil {
|
||||
return nil
|
||||
}
|
||||
cfg, err := ReadConfigFile[ModelConfig](galleryFile)
|
||||
if err != nil {
|
||||
xlog.Warn("Failed to read gallery file for installed model files", "model", name, "file", galleryFile, "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
files := make([]string, 0, len(cfg.Files))
|
||||
for _, f := range cfg.Files {
|
||||
// VerifyPath joins its argument onto modelsPath itself, so it must
|
||||
// get the relative name; an absolute path would always pass.
|
||||
if err := utils.VerifyPath(f.Filename, modelsPath); err != nil {
|
||||
xlog.Warn("Ignoring declared model file outside the models path", "model", name, "file", f.Filename)
|
||||
continue
|
||||
}
|
||||
files = append(files, filepath.Join(modelsPath, f.Filename))
|
||||
}
|
||||
return files
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package gallery_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
)
|
||||
|
||||
var _ = Describe("InstalledModelFiles", func() {
|
||||
var modelsPath string
|
||||
|
||||
BeforeEach(func() {
|
||||
modelsPath = GinkgoT().TempDir()
|
||||
})
|
||||
|
||||
writeGalleryFile := func(name, body string) {
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, gallery.GalleryFileName(name)), []byte(body), 0o644)).To(Succeed())
|
||||
}
|
||||
|
||||
It("returns the files the install declared, under the models path", func() {
|
||||
writeGalleryFile("big", `
|
||||
name: big
|
||||
files:
|
||||
- filename: llama-cpp/models/big/Big-00001-of-00002.gguf
|
||||
uri: huggingface://org/repo/Big-00001-of-00002.gguf
|
||||
- filename: llama-cpp/models/big/Big-00002-of-00002.gguf
|
||||
uri: huggingface://org/repo/Big-00002-of-00002.gguf
|
||||
`)
|
||||
Expect(gallery.InstalledModelFiles(modelsPath, "big")).To(Equal([]string{
|
||||
filepath.Join(modelsPath, "llama-cpp/models/big/Big-00001-of-00002.gguf"),
|
||||
filepath.Join(modelsPath, "llama-cpp/models/big/Big-00002-of-00002.gguf"),
|
||||
}))
|
||||
})
|
||||
|
||||
It("drops entries that escape the models path", func() {
|
||||
writeGalleryFile("evil", `
|
||||
files:
|
||||
- filename: ../outside.gguf
|
||||
- filename: ok.gguf
|
||||
`)
|
||||
Expect(gallery.InstalledModelFiles(modelsPath, "evil")).To(Equal([]string{
|
||||
filepath.Join(modelsPath, "ok.gguf"),
|
||||
}))
|
||||
})
|
||||
|
||||
It("returns nothing for a model that was not installed from a gallery", func() {
|
||||
Expect(gallery.InstalledModelFiles(modelsPath, "handwritten")).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
@@ -808,8 +808,11 @@ func GetLocalModelConfiguration(basePath string, name string) (*ModelConfig, err
|
||||
|
||||
func listModelFiles(systemState *system.SystemState, name string) ([]string, error) {
|
||||
|
||||
// VerifyPath joins its argument onto the models path itself, so every
|
||||
// check below passes the relative name: an already-joined absolute path
|
||||
// always lands inside the base and the check would pass anything.
|
||||
configFile := filepath.Join(systemState.Model.ModelsPath, fmt.Sprintf("%s.yaml", name))
|
||||
if err := utils.VerifyPath(configFile, systemState.Model.ModelsPath); err != nil {
|
||||
if err := utils.VerifyPath(fmt.Sprintf("%s.yaml", name), systemState.Model.ModelsPath); err != nil {
|
||||
return nil, fmt.Errorf("failed to verify path %s: %w", configFile, err)
|
||||
}
|
||||
|
||||
@@ -817,7 +820,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
|
||||
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
|
||||
|
||||
galleryFile := filepath.Join(systemState.Model.ModelsPath, galleryFileName(name))
|
||||
if err := utils.VerifyPath(galleryFile, systemState.Model.ModelsPath); err != nil {
|
||||
if err := utils.VerifyPath(galleryFileName(name), systemState.Model.ModelsPath); err != nil {
|
||||
return nil, fmt.Errorf("failed to verify path %s: %w", galleryFile, err)
|
||||
}
|
||||
|
||||
@@ -847,7 +850,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
|
||||
if err == nil && galleryconfig != nil {
|
||||
for _, f := range galleryconfig.Files {
|
||||
fullPath := filepath.Join(systemState.Model.ModelsPath, f.Filename)
|
||||
if err := utils.VerifyPath(fullPath, systemState.Model.ModelsPath); err != nil {
|
||||
if err := utils.VerifyPath(f.Filename, systemState.Model.ModelsPath); err != nil {
|
||||
return allFiles, fmt.Errorf("failed to verify path %s: %w", fullPath, err)
|
||||
}
|
||||
allFiles = append(allFiles, fullPath)
|
||||
@@ -858,7 +861,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
|
||||
|
||||
for _, f := range additionalFiles {
|
||||
fullPath := filepath.Join(filepath.Join(systemState.Model.ModelsPath, f))
|
||||
if err := utils.VerifyPath(fullPath, systemState.Model.ModelsPath); err != nil {
|
||||
if err := utils.VerifyPath(f, systemState.Model.ModelsPath); err != nil {
|
||||
return allFiles, fmt.Errorf("failed to verify path %s: %w", fullPath, err)
|
||||
}
|
||||
allFiles = append(allFiles, fullPath)
|
||||
|
||||
@@ -114,6 +114,11 @@ type RegisterNodeRequest struct {
|
||||
// VRAMBudget is the worker's operator-set VRAM cap ("80%" or "12GB"). The
|
||||
// registry resolves and enforces it against the raw reported VRAM.
|
||||
VRAMBudget string `json:"vram_budget,omitempty"`
|
||||
// Version is the LocalAI build version reported by the worker at
|
||||
// registration. Empty for workers registered before this field existed.
|
||||
Version string `json:"version,omitempty"`
|
||||
// Commit is the git commit hash the worker binary was built from.
|
||||
Commit string `json:"commit,omitempty"`
|
||||
}
|
||||
|
||||
// RegisterNodeEndpoint registers a new backend node.
|
||||
@@ -191,6 +196,8 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au
|
||||
Capability: req.Capability,
|
||||
MaxReplicasPerModel: maxReplicasPerModel,
|
||||
VRAMBudget: req.VRAMBudget,
|
||||
Version: req.Version,
|
||||
Commit: req.Commit,
|
||||
}
|
||||
|
||||
ctx := c.Request().Context()
|
||||
|
||||
@@ -9966,6 +9966,13 @@ button.collapsible-header:focus-visible {
|
||||
.node-inspector__actions .btn { justify-content: center; min-width: 0; }
|
||||
.node-inspector__back { align-items: center; background: transparent; border: 0; color: var(--color-primary); cursor: pointer; display: flex; font: inherit; font-size: var(--text-xs); gap: 6px; max-width: 285px; overflow: hidden; padding: 3px 0; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.node-inspector__back:focus-visible { border-radius: var(--radius-sm); outline: 2px solid var(--color-primary); outline-offset: 3px; }
|
||||
.node-inspector__models { margin: 10px 0 0; }
|
||||
.node-inspector__models > dd { margin: 0; }
|
||||
.node-inspector__model-list { display: grid; gap: 4px; list-style: none; margin: 6px 0 0; padding: 0; }
|
||||
.node-inspector__model-row { align-items: center; display: flex; flex-wrap: wrap; gap: 6px; font-size: var(--text-xs); }
|
||||
.node-inspector__model-row .cell-mono { font-family: var(--font-mono); font-size: .625rem; overflow-wrap: anywhere; }
|
||||
.node-inspector__model-row .state-pill { border-radius: var(--radius-full); font-size: .5625rem; font-weight: 600; padding: 1px 7px; text-transform: capitalize; }
|
||||
.node-inspector__model-row .text-muted { font-size: .5625rem; }
|
||||
.model-inspector__backends { margin-top: 10px; }
|
||||
.model-inspector__nodes { display: grid; gap: 9px; }
|
||||
.model-inspector__node { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-md); padding: 10px; }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import StatusPill from './StatusPill'
|
||||
import { formatBytes, formatCapacity, timeAgo } from './nodeStatus'
|
||||
import { formatBytes, formatCapacity, timeAgo, modelStateConfig } from './nodeStatus'
|
||||
import { nodesApi } from '../../utils/api'
|
||||
import { capacityReading, nodeLifecycleAction } from '../../utils/nodeFleet'
|
||||
import useInspectorDrawer from './useInspectorDrawer'
|
||||
@@ -24,6 +24,8 @@ function ResourceBar({ label, total, available, tone }) {
|
||||
export default function NodeInspector({ node, open, onClose, onApprove, onDrain, onResume, onBack, backLabel }) {
|
||||
const [backends, setBackends] = useState(null)
|
||||
const [backendError, setBackendError] = useState('')
|
||||
const [models, setModels] = useState(null)
|
||||
const [modelError, setModelError] = useState('')
|
||||
const nodeId = node?.id
|
||||
const backRef = useRef(null)
|
||||
const closeRef = useRef(null)
|
||||
@@ -48,6 +50,19 @@ export default function NodeInspector({ node, open, onClose, onApprove, onDrain,
|
||||
return () => { current = false }
|
||||
}, [open, nodeId])
|
||||
|
||||
useEffect(() => {
|
||||
if (!open || !nodeId) return undefined
|
||||
let current = true
|
||||
setModels(null)
|
||||
setModelError('')
|
||||
nodesApi.getModels(nodeId).then(data => {
|
||||
if (current) setModels(Array.isArray(data) ? data : [])
|
||||
}).catch(error => {
|
||||
if (current) setModelError(error.message || 'Unable to load models')
|
||||
})
|
||||
return () => { current = false }
|
||||
}, [open, nodeId])
|
||||
|
||||
if (!open || !node) return null
|
||||
const cpuKnown = node.cpu_logical_cores > 0 && Number.isFinite(node.cpu_usage_percent) && Number.isFinite(node.cpu_load_1)
|
||||
const disk = capacityReading(node.total_disk, node.available_disk)
|
||||
@@ -74,6 +89,7 @@ export default function NodeInspector({ node, open, onClose, onApprove, onDrain,
|
||||
<h3>Node</h3>
|
||||
<dl className="node-inspector__metrics">
|
||||
<InspectorMetric label="Address"><span className="node-inspector__address">{node.address || 'No address reported'}</span></InspectorMetric>
|
||||
<InspectorMetric label="Version">{node.version || '—'}</InspectorMetric>
|
||||
<InspectorMetric label="Heartbeat">{timeAgo(node.last_heartbeat)}</InspectorMetric>
|
||||
</dl>
|
||||
<div className="node-inspector__labels" aria-label="Node labels">{Object.keys(node.labels || {}).length ? Object.entries(node.labels).map(([key, value]) => <span key={key}>{key}={value}</span>) : <span className="text-muted">No labels</span>}</div>
|
||||
@@ -94,6 +110,26 @@ export default function NodeInspector({ node, open, onClose, onApprove, onDrain,
|
||||
<InspectorMetric label="Backends">{backendError ? <span className="text-error">{backendError}</span> : backends === null ? 'Loading…' : `${backends.length} backend${backends.length === 1 ? '' : 's'}`}</InspectorMetric>
|
||||
<InspectorMetric label="In-flight work">{node.in_flight_count ?? 0}</InspectorMetric>
|
||||
</dl>
|
||||
<div className="node-inspector__models">
|
||||
<dt className="drawer-eyebrow">Running models</dt>
|
||||
<dd>
|
||||
{modelError ? <span className="text-error">{modelError}</span>
|
||||
: models === null ? <span className="text-muted">Loading…</span>
|
||||
: models.length === 0 ? <span className="text-muted">No models loaded</span>
|
||||
: <ul className="node-inspector__model-list">
|
||||
{models.map(model => {
|
||||
const stCfg = modelStateConfig[model.state] || modelStateConfig.idle
|
||||
return (
|
||||
<li key={model.id || `${model.model_name}#${model.replica_index}`} className="node-inspector__model-row">
|
||||
<span className="cell-mono">{model.model_name}</span>
|
||||
<span className="state-pill" style={{ background: stCfg.bg, color: stCfg.color, border: `1px solid ${stCfg.border}` }}>{model.state}</span>
|
||||
<span className="text-muted">{model.in_flight ?? 0} in flight</span>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>}
|
||||
</dd>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<footer className="node-inspector__actions">
|
||||
|
||||
@@ -138,6 +138,10 @@ export default function NodeDetail() {
|
||||
<div className="drawer-eyebrow">In-flight</div>
|
||||
<span className="cell-mono">{node.in_flight_count || 0}</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="drawer-eyebrow">Version</div>
|
||||
<span className="cell-mono">{node.version || '—'}</span>
|
||||
</div>
|
||||
<div>
|
||||
<div className="drawer-eyebrow">Heartbeat</div>
|
||||
<span>{timeAgo(node.last_heartbeat)}</span>
|
||||
|
||||
@@ -93,7 +93,7 @@ func (s *ConfigService) GetConfig(_ context.Context, name string) (*ConfigView,
|
||||
if configPath == "" {
|
||||
return nil, ErrConfigFileMissing
|
||||
}
|
||||
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
|
||||
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
|
||||
}
|
||||
data, err := os.ReadFile(configPath)
|
||||
@@ -137,7 +137,7 @@ func (s *ConfigService) patchConfig(ctx context.Context, name string, patch map[
|
||||
return nil, fmt.Errorf("%w: PATCH cannot rename model %q to %q; use the model edit endpoint", ErrInvalidConfig, name, patchedName)
|
||||
}
|
||||
configPath := cfg.GetModelConfigFile()
|
||||
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
|
||||
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
|
||||
}
|
||||
diskYAML, err := os.ReadFile(configPath)
|
||||
@@ -289,7 +289,7 @@ func (s *ConfigService) editYAML(ctx context.Context, name string, body []byte)
|
||||
|
||||
configPath := existing.GetModelConfigFile()
|
||||
modelsPath := s.modelsPath()
|
||||
if err := utils.VerifyPath(configPath, modelsPath); err != nil {
|
||||
if err := utils.VerifyResolvedPath(configPath, modelsPath); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
|
||||
}
|
||||
|
||||
@@ -304,7 +304,7 @@ func (s *ConfigService) editYAML(ctx context.Context, name string, body []byte)
|
||||
}
|
||||
newConfigPath := filepath.Join(modelsPath, req.Name+".yaml")
|
||||
paths = append(paths, newConfigPath, filepath.Join(modelsPath, gallery.GalleryFileName(name)), filepath.Join(modelsPath, gallery.GalleryFileName(req.Name)))
|
||||
if err := utils.VerifyPath(newConfigPath, modelsPath); err != nil {
|
||||
if err := utils.VerifyPath(req.Name+".yaml", modelsPath); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
|
||||
}
|
||||
if _, err := os.Stat(newConfigPath); err == nil {
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package modeladmin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// A model config can be loaded from outside the models directory (for
|
||||
// example with --config-file). The admin mutations write the config file
|
||||
// back, so they must refuse a file outside the models directory rather than
|
||||
// write wherever the loader found it.
|
||||
var _ = Describe("ConfigService config file containment", func() {
|
||||
var (
|
||||
svc *ConfigService
|
||||
ctx context.Context
|
||||
outside string
|
||||
orig []byte
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
svc, _ = newTestService()
|
||||
ctx = context.Background()
|
||||
outside = filepath.Join(GinkgoT().TempDir(), "external.yaml")
|
||||
orig = []byte("name: external\nbackend: llama-cpp\n")
|
||||
Expect(os.WriteFile(outside, orig, 0o644)).To(Succeed())
|
||||
Expect(svc.Loader.ReadModelConfig(outside, svc.AppConfig.ToConfigLoaderOptions()...)).To(Succeed())
|
||||
cfg, ok := svc.Loader.GetModelConfig("external")
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(cfg.GetModelConfigFile()).To(Equal(outside))
|
||||
})
|
||||
|
||||
It("refuses to pin a model whose config file is outside the models directory", func() {
|
||||
_, err := svc.TogglePinned(ctx, "external", ActionPin, nil)
|
||||
Expect(err).To(MatchError(ErrPathNotTrusted))
|
||||
Expect(os.ReadFile(outside)).To(Equal(orig))
|
||||
})
|
||||
|
||||
It("refuses to toggle the state of such a model", func() {
|
||||
_, err := svc.ToggleState(ctx, "external", ActionDisable)
|
||||
Expect(err).To(MatchError(ErrPathNotTrusted))
|
||||
Expect(os.ReadFile(outside)).To(Equal(orig))
|
||||
})
|
||||
|
||||
It("refuses to patch such a model", func() {
|
||||
_, err := svc.PatchConfig(ctx, "external", map[string]any{"context_size": 4096})
|
||||
Expect(err).To(MatchError(ErrPathNotTrusted))
|
||||
Expect(os.ReadFile(outside)).To(Equal(orig))
|
||||
})
|
||||
|
||||
It("refuses to read such a model's config", func() {
|
||||
_, err := svc.GetConfig(ctx, "external")
|
||||
Expect(err).To(MatchError(ErrPathNotTrusted))
|
||||
})
|
||||
})
|
||||
@@ -29,7 +29,7 @@ func (s *ConfigService) TogglePinned(_ context.Context, name string, action Acti
|
||||
if configPath == "" {
|
||||
return nil, ErrConfigFileMissing
|
||||
}
|
||||
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
|
||||
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
|
||||
}
|
||||
if err := mutateYAMLBoolFlag(configPath, "pinned", action == ActionPin); err != nil {
|
||||
|
||||
@@ -49,7 +49,7 @@ func (s *ConfigService) toggleState(ctx context.Context, name string, action Act
|
||||
if configPath == "" {
|
||||
return nil, ErrConfigFileMissing
|
||||
}
|
||||
if err := utils.VerifyPath(configPath, s.modelsPath()); err != nil {
|
||||
if err := utils.VerifyResolvedPath(configPath, s.modelsPath()); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrPathNotTrusted, err)
|
||||
}
|
||||
var result *ToggleResult
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
|
||||
// declaredExtraFiles returns the files the model's install declared that the
|
||||
// path fields of opts do not already stage: neither named by a field nor
|
||||
// inside a directory a field names. It must run on the local paths, before
|
||||
// staging rewrites the fields to remote ones.
|
||||
func (r *SmartRouter) declaredExtraFiles(trackingKey string, opts *pb.ModelOptions) []string {
|
||||
if r.modelFiles == nil || opts == nil || trackingKey == "" {
|
||||
return nil
|
||||
}
|
||||
covered := append([]string{
|
||||
opts.ModelFile, opts.MMProj, opts.LoraAdapter, opts.DraftModel,
|
||||
opts.CLIPModel, opts.Tokenizer, opts.AudioPath, opts.LoraBase,
|
||||
}, opts.LoraAdapters...)
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
var extra []string
|
||||
for _, p := range r.modelFiles(trackingKey) {
|
||||
p = filepath.Clean(p)
|
||||
if _, dup := seen[p]; dup || coveredByField(p, covered) {
|
||||
continue
|
||||
}
|
||||
seen[p] = struct{}{}
|
||||
extra = append(extra, p)
|
||||
}
|
||||
return extra
|
||||
}
|
||||
|
||||
func coveredByField(path string, fields []string) bool {
|
||||
for _, f := range fields {
|
||||
if f == "" {
|
||||
continue
|
||||
}
|
||||
f = filepath.Clean(f)
|
||||
if path == f || strings.HasPrefix(path, f+string(filepath.Separator)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// existingFiles drops declared files that are not on the frontend. An install
|
||||
// can declare files that are gone by load time (an archive unpacked and then
|
||||
// removed, say), so a missing one is not a reason to refuse the load; the
|
||||
// backend reports it if it really needed it.
|
||||
func existingFiles(paths []string, nodeName, trackingKey string) []string {
|
||||
out := paths[:0:0]
|
||||
for _, p := range paths {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
xlog.Warn("Skipping staging for declared model file that is not on the frontend", "path", p, "node", nodeName, "model", trackingKey, "error", err)
|
||||
continue
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stagingPayloadBytes totals the on-disk size of everything staging uploads
|
||||
// for a model: the path fields plus the declared files they do not cover. The
|
||||
// first shard of a split GGUF can be a few MB of metadata while the weights
|
||||
// sit in the others, so sizing the fields alone starves the load budget and
|
||||
// the disk-headroom check.
|
||||
func (r *SmartRouter) stagingPayloadBytes(trackingKey string, opts *pb.ModelOptions) int64 {
|
||||
total := modelPayloadBytes(opts)
|
||||
for _, p := range r.declaredExtraFiles(trackingKey, opts) {
|
||||
total += pathBytes(p)
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -124,6 +124,11 @@ type BackendNode struct {
|
||||
// worker's re-registration value does not clobber it (mirrors
|
||||
// MaxReplicasPerModelManuallySet).
|
||||
VRAMBudgetManuallySet bool `gorm:"column:vram_budget_manually_set;default:false" json:"vram_budget_manually_set"`
|
||||
// Version is the LocalAI build version reported by the worker at
|
||||
// registration. Empty for workers registered before this field existed.
|
||||
Version string `gorm:"column:version;size:64" json:"version,omitempty"`
|
||||
// Commit is the git commit hash the worker binary was built from.
|
||||
Commit string `gorm:"column:commit;size:64" json:"commit,omitempty"`
|
||||
APIKeyID string `gorm:"size:36" json:"-"` // auto-provisioned API key ID (for cleanup)
|
||||
AuthUserID string `gorm:"size:36" json:"-"` // auto-provisioned user ID (for cleanup)
|
||||
LastHeartbeat time.Time `gorm:"column:last_heartbeat" json:"last_heartbeat"`
|
||||
|
||||
@@ -73,6 +73,12 @@ type SmartRouterOptions struct {
|
||||
// nil disables the exclusion. Deliberate teardown (UnloadModel, admin
|
||||
// endpoints, node drain) is unaffected.
|
||||
PinnedResolver PinnedModelResolver
|
||||
// ModelFiles, when set, returns the absolute local paths of every file a
|
||||
// model's install declared (gallery `files:`, config `download_files`).
|
||||
// The path fields of a load request name only what the backend opens
|
||||
// first; this is how staging learns about the rest, such as the other
|
||||
// shards of a split GGUF. nil stages the path fields alone.
|
||||
ModelFiles func(modelName string) []string
|
||||
// PrefixProvider, when set, enables prefix-cache-aware routing: requests
|
||||
// carrying a prompt prefix chain (distributedhdr.PrefixChain) are biased
|
||||
// toward the node that already holds the longest matching prefix, subject
|
||||
@@ -189,6 +195,9 @@ type SmartRouter struct {
|
||||
// pinnedResolver feeds the eviction paths the set of pinned model names
|
||||
// (see SmartRouterOptions.PinnedResolver). nil disables the exclusion.
|
||||
pinnedResolver PinnedModelResolver
|
||||
// modelFiles resolves a model's declared files (see
|
||||
// SmartRouterOptions.ModelFiles). nil stages the path fields alone.
|
||||
modelFiles func(modelName string) []string
|
||||
// prefixProvider is the prefix-cache routing seam (nil disables it; see
|
||||
// SmartRouterOptions.PrefixProvider). prefixConfig holds the global policy
|
||||
// and thresholds.
|
||||
@@ -283,6 +292,7 @@ func NewSmartRouter(registry ModelRouter, opts SmartRouterOptions) *SmartRouter
|
||||
stagingTracker: NewStagingTracker(),
|
||||
conflictResolver: opts.ConflictResolver,
|
||||
pinnedResolver: opts.PinnedResolver,
|
||||
modelFiles: opts.ModelFiles,
|
||||
probeCache: newProbeCache(probeCacheTTL),
|
||||
prefixProvider: opts.PrefixProvider,
|
||||
prefixConfig: opts.PrefixConfig,
|
||||
@@ -425,7 +435,7 @@ func (r *SmartRouter) scheduleAndLoad(ctx context.Context, backendType, tracking
|
||||
// Size the remote load budget BEFORE staging: stageModelFiles rewrites the
|
||||
// path fields to their remote equivalents on a clone, and only the local
|
||||
// paths can be stat'ed here.
|
||||
payloadBytes := modelPayloadBytes(modelOpts)
|
||||
payloadBytes := r.stagingPayloadBytes(trackingKey, modelOpts)
|
||||
loadTimeout := r.loadTimeoutFor(payloadBytes)
|
||||
|
||||
// Pre-stage model files via FileStager before loading
|
||||
@@ -1367,7 +1377,7 @@ func (r *SmartRouter) narrowByDiskHeadroom(ctx context.Context, modelID string,
|
||||
return candidateNodeIDs, nil
|
||||
}
|
||||
|
||||
requiredDisk := DiskRequirementFor(modelPayloadBytes(modelOpts))
|
||||
requiredDisk := DiskRequirementFor(r.stagingPayloadBytes(modelID, modelOpts))
|
||||
diskCandidates, diskErr := r.registry.NarrowByDiskHeadroom(ctx, candidateNodeIDs, requiredDisk)
|
||||
|
||||
// The check runs even when disabled. "Disabled" means do not BLOCK, not do
|
||||
@@ -1568,6 +1578,10 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
|
||||
localModelDir = filepath.Dir(opts.ModelFile)
|
||||
}
|
||||
|
||||
// Resolved before the path fields are rewritten to remote paths below,
|
||||
// since that is what tells which declared files the fields already cover.
|
||||
declared := existingFiles(r.declaredExtraFiles(trackingKey, opts), node.Name, trackingKey)
|
||||
|
||||
// keyMapper generates storage keys namespaced under trackingKey, preserving
|
||||
// subdirectory structure relative to frontendModelsDir. This ensures:
|
||||
// 1. All files for a model land in one directory on the worker for clean deletion
|
||||
@@ -1614,6 +1628,7 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
|
||||
totalFiles++
|
||||
}
|
||||
}
|
||||
totalFiles += len(declared)
|
||||
|
||||
// Start tracking staging progress
|
||||
r.stagingTracker.Start(trackingKey, node.Name, totalFiles)
|
||||
@@ -1757,6 +1772,21 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
|
||||
}
|
||||
}
|
||||
|
||||
for _, localPath := range declared {
|
||||
fileIdx++
|
||||
fileName := filepath.Base(localPath)
|
||||
stageCtx := r.withStagingCallback(ctx, trackingKey, fileName, fileIdx, totalFiles)
|
||||
|
||||
xlog.Info("Staging declared model file", "model", trackingKey, "node", node.Name, "file", fileName, "fileIndex", fileIdx, "totalFiles", totalFiles)
|
||||
if _, err := r.fileStager.EnsureRemote(stageCtx, node.ID, localPath, keyMapper.Key(localPath)); err != nil {
|
||||
// The install declared it, so the backend may read it: loading
|
||||
// without it fails later with a less useful error.
|
||||
xlog.Error("Failed to stage declared model file for remote node", "node", node.Name, "path", localPath, "error", err)
|
||||
return nil, fmt.Errorf("staging declared model file %s: %w", localPath, err)
|
||||
}
|
||||
r.stagingTracker.FileComplete(trackingKey, fileIdx, totalFiles)
|
||||
}
|
||||
|
||||
// Stage file paths referenced in generic Options (key:value pairs where values
|
||||
// are file paths). Options stay as relative paths — backends resolve them via ModelPath.
|
||||
for _, options := range [][]string{opts.Options, opts.Overrides} {
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||
)
|
||||
|
||||
// A model's config names only the file the backend opens first, but its
|
||||
// install can declare more that the backend reads by itself: llama.cpp opens
|
||||
// the "-0000N-of-0000M" shards of a split GGUF from the directory of the first
|
||||
// one. The worker has no view of the frontend's models directory, so every
|
||||
// declared file must be staged, or the load fails with "failed to load GGUF
|
||||
// split".
|
||||
var _ = Describe("stageModelFiles declared model files", func() {
|
||||
var (
|
||||
stager *fakeFileStager
|
||||
router *SmartRouter
|
||||
node *BackendNode
|
||||
modelDir string
|
||||
shards []string
|
||||
mmproj string
|
||||
declared map[string][]string
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
stager = &fakeFileStager{}
|
||||
declared = map[string][]string{}
|
||||
router = &SmartRouter{
|
||||
fileStager: stager,
|
||||
stagingTracker: NewStagingTracker(),
|
||||
modelFiles: func(name string) []string { return declared[name] },
|
||||
}
|
||||
node = &BackendNode{ID: "node-1", Name: "node-1", Address: "10.0.0.1:50051"}
|
||||
root := GinkgoT().TempDir()
|
||||
modelDir = filepath.Join(root, "llama-cpp", "models", "big")
|
||||
Expect(os.MkdirAll(modelDir, 0o755)).To(Succeed())
|
||||
|
||||
shards = nil
|
||||
for _, name := range []string{
|
||||
"Big-Q4_K_M-00001-of-00003.gguf",
|
||||
"Big-Q4_K_M-00002-of-00003.gguf",
|
||||
"Big-Q4_K_M-00003-of-00003.gguf",
|
||||
} {
|
||||
p := filepath.Join(modelDir, name)
|
||||
Expect(os.WriteFile(p, []byte("shard "+name), 0o644)).To(Succeed())
|
||||
shards = append(shards, p)
|
||||
}
|
||||
mmproj = filepath.Join(root, "llama-cpp", "mmproj", "big", "mmproj.gguf")
|
||||
Expect(os.MkdirAll(filepath.Dir(mmproj), 0o755)).To(Succeed())
|
||||
Expect(os.WriteFile(mmproj, []byte("mmproj"), 0o644)).To(Succeed())
|
||||
})
|
||||
|
||||
opts := func() *pb.ModelOptions {
|
||||
return &pb.ModelOptions{
|
||||
Model: "llama-cpp/models/big/Big-Q4_K_M-00001-of-00003.gguf",
|
||||
ModelFile: shards[0],
|
||||
MMProj: mmproj,
|
||||
}
|
||||
}
|
||||
|
||||
stagedPaths := func() []string {
|
||||
out := make([]string, 0, len(stager.ensureCalls))
|
||||
for _, c := range stager.ensureCalls {
|
||||
out = append(out, c.localPath)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
It("stages every declared file once, beside the ones the config names", func() {
|
||||
declared["big"] = append(append([]string{}, shards...), mmproj)
|
||||
|
||||
staged, err := router.stageModelFiles(context.Background(), node, opts(), "big")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(stagedPaths()).To(ConsistOf(shards[0], mmproj, shards[1], shards[2]))
|
||||
|
||||
// llama.cpp derives the other shards' paths from the first one, so
|
||||
// they must land in the same remote directory.
|
||||
for _, c := range stager.ensureCalls {
|
||||
if c.localPath != mmproj {
|
||||
Expect(filepath.Dir(c.key)).To(Equal(filepath.Dir(stager.ensureCalls[0].key)))
|
||||
}
|
||||
}
|
||||
Expect(staged.ModelFile).To(Equal("/remote/" + stager.ensureCalls[0].key))
|
||||
})
|
||||
|
||||
It("sizes declared files for the load budget and disk check", func() {
|
||||
declared["big"] = append(append([]string{}, shards...), mmproj)
|
||||
|
||||
var want int64
|
||||
for _, p := range append(append([]string{}, shards...), mmproj) {
|
||||
fi, err := os.Stat(p)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
want += fi.Size()
|
||||
}
|
||||
Expect(router.stagingPayloadBytes("big", opts())).To(Equal(want))
|
||||
})
|
||||
|
||||
It("skips a declared file that is missing locally instead of failing", func() {
|
||||
declared["big"] = append(append([]string{}, shards...), filepath.Join(modelDir, "gone.bin"))
|
||||
|
||||
_, err := router.stageModelFiles(context.Background(), node, opts(), "big")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(stagedPaths()).To(ConsistOf(shards[0], mmproj, shards[1], shards[2]))
|
||||
})
|
||||
|
||||
It("does not stage a declared file twice when a directory field covers it", func() {
|
||||
declared["dir"] = []string{shards[1]}
|
||||
|
||||
_, err := router.stageModelFiles(context.Background(), node,
|
||||
&pb.ModelOptions{Model: "llama-cpp/models/big", ModelFile: modelDir}, "dir")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(stagedPaths()).To(ConsistOf(shards[0], shards[1], shards[2]))
|
||||
})
|
||||
|
||||
It("stages only the named files for a model that declares none", func() {
|
||||
_, err := router.stageModelFiles(context.Background(), node, opts(), "handwritten")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(stagedPaths()).To(ConsistOf(shards[0], mmproj))
|
||||
})
|
||||
})
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mudler/LocalAI/internal"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||
"github.com/mudler/xlog"
|
||||
@@ -154,6 +155,8 @@ func (cfg *Config) registrationBody() map[string]any {
|
||||
"gpu_compute_capability": gpuComputeCap,
|
||||
"capability": capability,
|
||||
"max_replicas_per_model": maxReplicas,
|
||||
"version": internal.Version,
|
||||
"commit": internal.Commit,
|
||||
}
|
||||
|
||||
// Report free space on the filesystem that backs the MODELS directory.
|
||||
|
||||
@@ -41,7 +41,7 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal
|
||||
// Check if it's a model gallery, or print a warning
|
||||
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity, installOptions...)
|
||||
if e != nil && found {
|
||||
xlog.Error("[startup] failed installing model", "error", err, "model", url)
|
||||
xlog.Error("[startup] failed installing model", "error", e, "model", url)
|
||||
err = errors.Join(err, e)
|
||||
} else if !found {
|
||||
xlog.Debug("[startup] model not found in the gallery", "model", url)
|
||||
@@ -54,7 +54,7 @@ func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.Gal
|
||||
modelConfig, discoverErr := importers.DiscoverModelConfig(url, json.RawMessage{})
|
||||
if discoverErr != nil {
|
||||
xlog.Error("[startup] failed to discover model config", "error", discoverErr, "model", url)
|
||||
err = errors.Join(discoverErr, fmt.Errorf("failed to discover model config: %w", err))
|
||||
err = errors.Join(err, fmt.Errorf("failed to discover model config: %w", discoverErr))
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user