feat: materialize Hugging Face model artifacts (#10825)

* feat(config): add model artifact source contract

Assisted-by: Codex:GPT-5 [Codex]

* feat(downloader): add authenticated raw-byte progress

Assisted-by: Codex:GPT-5 [Codex]

* feat(huggingface): resolve immutable snapshot manifests

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): add artifact storage primitives

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): materialize pinned Hugging Face snapshots

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): bind managed snapshots at runtime

Assisted-by: Codex:GPT-5 [Codex]

* feat(gallery): materialize model artifacts during install

Assisted-by: Codex:GPT-5 [Codex]

* feat(gallery): declare managed Hugging Face artifacts

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): preload managed model artifacts

Assisted-by: Codex:GPT-5 [Codex]

* fix(gallery): retain shared artifact caches on delete

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): report artifact acquisition progress

Assisted-by: Codex:GPT-5 [Codex]

* refactor(backends): load managed models from ModelFile

Assisted-by: Codex:GPT-5 [Codex]

* refactor(backends): load staged speech model snapshots

Assisted-by: Codex:GPT-5 [Codex]

* refactor(backends): use staged snapshots in engine backends

Assisted-by: Codex:GPT-5 [Codex]

* test(distributed): cover staged artifact snapshots

Assisted-by: Codex:GPT-5 [Codex]

* docs: explain managed model artifacts

Assisted-by: Codex:GPT-5 [Codex]

* docs: add product design context

Assisted-by: Codex:GPT-5 [Codex]

* feat(ui): show model artifact download progress

Assisted-by: Codex:GPT-5 [Codex]

* Eagerly materialize Hugging Face artifacts

Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback.

Assisted-by: Codex:GPT-5 [shell]

* Refactor HF
  downloads through a shared executor

Assisted-by: Codex:GPT-5 [shell]

* drop

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
2026-07-15 01:09:33 +02:00
committed by GitHub
parent d82c38ee77
commit bcc41219f7
83 changed files with 4315 additions and 339 deletions

View File

@@ -116,7 +116,11 @@ func newApplication(appConfig *config.ApplicationConfig) *Application {
ml.SetLoadObserver(corebackend.ModelLoadTraceObserver(appConfig))
app := &Application{
backendLoader: config.NewModelConfigLoader(appConfig.SystemState.Model.ModelsPath),
backendLoader: config.NewModelConfigLoader(
appConfig.SystemState.Model.ModelsPath,
config.WithArtifactMaterializer(appConfig.ModelArtifactMaterializer),
config.WithPreloadDisplay(appConfig.ModelPreloadRenderMode, appConfig.DisableModelPreloadColor),
),
modelLoader: ml,
applicationConfig: appConfig,
templatesEvaluator: templates.NewEvaluator(appConfig.SystemState.Model.ModelsPath),

View File

@@ -0,0 +1,55 @@
package application
import (
"context"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
)
type applicationLoaderMaterializer struct {
seen []modelartifacts.Spec
}
func (f *applicationLoaderMaterializer) Ensure(_ context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
f.seen = append(f.seen, spec)
spec.Resolved = &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
}
return modelartifacts.Result{Spec: spec}, nil
}
var _ = Describe("application model artifact wiring", func() {
It("injects the controller materializer into the model config loader", func() {
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(
system.WithModelPath(modelsPath),
system.WithBackendPath(GinkgoT().TempDir()),
)
Expect(err).NotTo(HaveOccurred())
materializer := &applicationLoaderMaterializer{}
app := newApplication(&config.ApplicationConfig{
Context: context.Background(),
SystemState: state,
ModelArtifactMaterializer: materializer,
})
Expect(os.WriteFile(filepath.Join(modelsPath, "managed.yaml"), []byte(`
name: managed
backend: transformers
artifacts:
- source: {type: huggingface, repo: owner/repo}
parameters: {model: owner/repo}
`), 0644)).To(Succeed())
Expect(app.ModelConfigLoader().LoadModelConfigsFromPath(modelsPath)).To(Succeed())
Expect(app.ModelConfigLoader().PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
Expect(materializer.seen).To(HaveLen(1))
})
})

View File

@@ -414,18 +414,18 @@ func New(opts ...config.AppOption) (*Application, error) {
}
}
if err := application.ModelConfigLoader().Preload(options.SystemState.Model.ModelsPath); err != nil {
if err := application.ModelConfigLoader().PreloadWithContext(options.Context, options.SystemState.Model.ModelsPath); err != nil {
xlog.Error("error downloading models", "error", err)
}
if options.PreloadJSONModels != "" {
if err := galleryop.ApplyGalleryFromString(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels, options.RequireBackendIntegrity); err != nil {
if err := galleryop.ApplyGalleryFromString(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels, options.RequireBackendIntegrity, gallery.WithArtifactMaterializer(options.ModelArtifactMaterializer)); err != nil {
return nil, err
}
}
if options.PreloadModelsFromPath != "" {
if err := galleryop.ApplyGalleryFromFile(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath, options.RequireBackendIntegrity); err != nil {
if err := galleryop.ApplyGalleryFromFile(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath, options.RequireBackendIntegrity, gallery.WithArtifactMaterializer(options.ModelArtifactMaterializer)); err != nil {
return nil, err
}
}

View File

@@ -109,7 +109,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
if !slices.Contains(modelNames, modelName) {
utils.ResetDownloadTimers()
// if we failed to load the model, we try to download it
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, modelName, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries, o.RequireBackendIntegrity)
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, modelName, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries, o.RequireBackendIntegrity, gallery.WithArtifactMaterializer(o.ModelArtifactMaterializer))
if err != nil {
xlog.Error("failed to install model from gallery", "error", err, "model", modelFile)
//return nil, err

View File

@@ -0,0 +1,47 @@
package backend
import (
"encoding/json"
"os"
"path/filepath"
"strings"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
var _ = Describe("managed model runtime options", func() {
It("estimates weights from the committed manifest without repository fallback", func() {
const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
modelsPath := GinkgoT().TempDir()
spec := modelartifacts.Spec{
Name: "model", Target: "model",
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: cacheKey,
},
}
relative, err := modelartifacts.RelativeSnapshotPath(cacheKey)
Expect(err).NotTo(HaveOccurred())
manifest := modelartifacts.Manifest{
Version: modelartifacts.ManifestVersion,
Artifact: spec,
Files: []modelartifacts.ManifestFile{{
Path: "weights/model.safetensors", Size: 12, SHA256: strings.Repeat("a", 64),
}},
}
encoded, err := json.Marshal(manifest)
Expect(err).NotTo(HaveOccurred())
manifestPath := filepath.Join(modelsPath, filepath.Dir(relative), "manifest.json")
Expect(os.MkdirAll(filepath.Dir(manifestPath), 0750)).To(Succeed())
Expect(os.WriteFile(manifestPath, encoded, 0644)).To(Succeed())
cfg := config.ModelConfig{Artifacts: []modelartifacts.Spec{spec}}
Expect(estimateModelSizeBytes(cfg, modelsPath)).To(Equal(int64(12)))
})
})

View File

@@ -12,9 +12,10 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/trace"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/downloader"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/vram"
"github.com/mudler/xlog"
)
@@ -93,8 +94,9 @@ func recordModelLoadFailure(appConfig *config.ApplicationConfig, modelName, back
func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
seen := make(map[string]bool)
input := vram.ModelEstimateInput{}
managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil
addFile := func(uri string) {
addFile := func(uri string, size int64) {
if !vram.IsWeightFile(uri) {
return
}
@@ -106,7 +108,7 @@ func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
return
}
seen[resolved] = true
input.Files = append(input.Files, vram.FileInput{URI: resolved})
input.Files = append(input.Files, vram.FileInput{URI: resolved, Size: size})
}
// tryHFRepo resolves any huggingface:// or hf:// URI to an HTTPS URL and
@@ -123,13 +125,25 @@ func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
for _, f := range c.DownloadFiles {
uriStr := string(f.URI)
addFile(uriStr)
tryHFRepo(uriStr)
addFile(uriStr, 0)
if !managedPrimary {
tryHFRepo(uriStr)
}
}
if managedPrimary {
relative := c.ModelFileName()
manifest, err := modelartifacts.ReadManifest(filepath.Join(modelsPath, filepath.Dir(relative), "manifest.json"))
if err == nil {
for _, file := range manifest.Files {
addFile(filepath.Join(relative, filepath.FromSlash(file.Path)), file.Size)
}
}
} else {
addFile(c.Model, 0)
tryHFRepo(c.Model)
}
addFile(c.Model)
tryHFRepo(c.Model)
if c.MMProj != "" {
addFile(c.MMProj)
addFile(c.MMProj, 0)
}
if len(input.Files) == 0 && input.HFRepo == "" {
@@ -153,6 +167,10 @@ func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...mo
model.WithContext(so.Context),
model.WithModelID(c.ModelID()),
}
managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil
if managedPrimary {
defOpts = append(defOpts, model.WithModelFile(c.ModelFileName()))
}
threads := 1

View File

@@ -14,6 +14,7 @@ import (
"github.com/mudler/LocalAI/core/startup"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/xlog"
"github.com/schollz/progressbar/v3"
@@ -24,6 +25,9 @@ type ModelsCMDFlags struct {
BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"`
ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"storage"`
Color string `env:"COLOR" hidden:""`
NoColor string `env:"NO_COLOR" hidden:""`
HFToken string `env:"HF_TOKEN" hidden:""`
}
type ModelsList struct {
@@ -80,10 +84,18 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
return err
}
artifactMaterializer := modelartifacts.NewDefaultManager(
modelartifacts.WithHuggingFaceToken(mi.HFToken),
)
galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{
SystemState: systemState,
SystemState: systemState,
ModelArtifactMaterializer: artifactMaterializer,
ModelPreloadRenderMode: mi.Color,
DisableModelPreloadColor: mi.NoColor != "",
}, model.NewModelLoader(systemState))
err = galleryService.Start(context.Background(), config.NewModelConfigLoader(mi.ModelsPath), systemState)
err = galleryService.Start(context.Background(), config.NewModelConfigLoader(mi.ModelsPath,
config.WithArtifactMaterializer(artifactMaterializer),
config.WithPreloadDisplay(mi.Color, mi.NoColor != "")), systemState)
if err != nil {
return err
}

View File

@@ -16,6 +16,7 @@ import (
"github.com/mudler/LocalAI/core/http"
"github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/internal"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/signals"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/xlog"
@@ -28,6 +29,9 @@ import (
type RunCMD struct {
ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"`
Color string `env:"COLOR" hidden:""`
NoColor string `env:"NO_COLOR" hidden:""`
HFToken string `env:"HF_TOKEN" hidden:""`
ExternalBackends []string `env:"LOCALAI_EXTERNAL_BACKENDS,EXTERNAL_BACKENDS" help:"A list of external backends to load from gallery on boot" group:"backends"`
WebRTCNAT1To1IPs []string `env:"LOCALAI_WEBRTC_NAT_1TO1_IPS,WEBRTC_NAT_1TO1_IPS" help:"IPs advertised as the host ICE candidates for /v1/realtime WebRTC instead of every local interface. Set to the reachable host/LAN IP when running under Docker host networking or NAT, where pion otherwise offers unreachable bridge addresses and the connection drops after ICE consent checks fail." group:"api"`
@@ -238,6 +242,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
opts := []config.AppOption{
config.WithContext(context.Background()),
config.WithModelArtifactMaterializer(modelartifacts.NewDefaultManager(
modelartifacts.WithHuggingFaceToken(r.HFToken),
)),
config.WithModelPreloadDisplay(r.Color, r.NoColor != ""),
config.WithConfigFile(r.ModelsConfigFile),
config.WithJSONStringPreload(r.PreloadModels),
config.WithYAMLConfigPreload(r.PreloadModelsConfig),

View File

@@ -0,0 +1,34 @@
package config
import (
"context"
"reflect"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
type applicationArtifactMaterializer struct{}
func (*applicationArtifactMaterializer) Ensure(context.Context, string, modelartifacts.Spec) (modelartifacts.Result, error) {
return modelartifacts.Result{}, nil
}
var _ = Describe("ApplicationConfig model artifact materializer", func() {
It("provides a default materializer", func() {
Expect(NewApplicationConfig().ModelArtifactMaterializer).NotTo(BeNil())
})
It("accepts an injected materializer without exposing it to serialization", func() {
materializer := &applicationArtifactMaterializer{}
appConfig := NewApplicationConfig(WithModelArtifactMaterializer(materializer))
Expect(appConfig.ModelArtifactMaterializer).To(BeIdenticalTo(materializer))
field, found := reflect.TypeOf(ApplicationConfig{}).FieldByName("ModelArtifactMaterializer")
Expect(found).To(BeTrue())
Expect(field.Tag.Get("json")).To(Equal("-"))
Expect(field.Tag.Get("yaml")).To(Equal("-"))
})
})

View File

@@ -7,6 +7,7 @@ import (
"time"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/mudler/xlog"
@@ -18,6 +19,13 @@ type ApplicationConfig struct {
SystemState *system.SystemState
ExternalBackends []string
// ModelArtifactMaterializer is a controller-only acquisition capability.
// It is excluded from serialization so credentials captured by its concrete
// implementation cannot enter persisted settings or distributed payloads.
ModelArtifactMaterializer ArtifactMaterializer `json:"-" yaml:"-"`
ModelPreloadRenderMode string `json:"-" yaml:"-"`
DisableModelPreloadColor bool `json:"-" yaml:"-"`
// WebRTCNAT1To1IPs, when set, are advertised as the host ICE candidates for
// /v1/realtime WebRTC instead of every local interface address. Needed when
// the routable address differs from what pion gathers — e.g. Docker host
@@ -246,9 +254,11 @@ type AppOption func(*ApplicationConfig)
func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
opt := &ApplicationConfig{
Context: context.Background(),
UploadLimitMB: 15,
Debug: true,
Context: context.Background(),
UploadLimitMB: 15,
Debug: true,
ModelArtifactMaterializer: modelartifacts.NewDefaultManager(),
ModelPreloadRenderMode: "dark",
// Capture backend process stdout/stderr into the per-model
// BackendLogStore by default so the UI "Backend Logs" page works out
// of the box in single mode, matching worker/distributed mode (which
@@ -256,9 +266,9 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
// toggle can still turn it off (a persisted false wins - see
// loadRuntimeSettingsFromFile).
EnableBackendLogging: true,
AgentJobRetentionDays: 30, // Default: 30 days
LRUEvictionMaxRetries: 30, // Default: 30 retries
LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
AgentJobRetentionDays: 30, // Default: 30 days
LRUEvictionMaxRetries: 30, // Default: 30 retries
LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
ModelLoadFailureCooldown: 10 * time.Second, // Default: 10s base cooldown after a failed load
// WatchDogInterval is intentionally left at the zero value here.
// The startup loader applies a persisted runtime_settings.json value
@@ -636,6 +646,25 @@ func WithContext(ctx context.Context) AppOption {
}
}
// WithModelArtifactMaterializer injects the controller-only model acquisition capability.
func WithModelArtifactMaterializer(materializer ArtifactMaterializer) AppOption {
return func(o *ApplicationConfig) {
if materializer != nil {
o.ModelArtifactMaterializer = materializer
}
}
}
// WithModelPreloadDisplay configures terminal rendering for model preload output.
func WithModelPreloadDisplay(renderMode string, disableColor bool) AppOption {
return func(o *ApplicationConfig) {
if renderMode != "" {
o.ModelPreloadRenderMode = renderMode
}
o.DisableModelPreloadColor = disableColor
}
}
func WithYAMLConfigPreload(configFile string) AppOption {
return func(o *ApplicationConfig) {
o.PreloadModelsFromPath = configFile

View File

@@ -82,6 +82,14 @@ func DefaultRegistry() map[string]FieldMetaOverride {
Options: ModalityOptions,
Order: 8,
},
"artifacts": {
Section: "general",
Label: "Managed Model Artifacts",
Description: "Controller-managed model sources. LocalAI resolves, downloads, verifies, and binds these sources before a backend loads.",
Component: "json-editor",
Advanced: true,
Order: 9,
},
// --- LLM ---
"context_size": {

View File

@@ -0,0 +1,110 @@
package config
import (
"fmt"
"os"
"path/filepath"
"gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
func persistArtifactBinding(fileName, modelName string, result modelartifacts.Result) error {
data, err := os.ReadFile(fileName)
if err != nil {
return err
}
var document yaml.Node
if err := yaml.Unmarshal(data, &document); err != nil {
return err
}
target, err := findModelMapping(&document, modelName)
if err != nil {
return err
}
artifactValue := &yaml.Node{}
encoded, err := yaml.Marshal([]modelartifacts.Spec{result.Spec})
if err != nil {
return err
}
if err := yaml.Unmarshal(encoded, artifactValue); err != nil {
return err
}
setMappingValue(target, "artifacts", artifactValue.Content[0])
updated, err := yaml.Marshal(&document)
if err != nil {
return err
}
return writeBindingAtomic(fileName, updated)
}
func findModelMapping(document *yaml.Node, modelName string) (*yaml.Node, error) {
if len(document.Content) != 1 {
return nil, fmt.Errorf("invalid model configuration document")
}
root := document.Content[0]
if root.Kind == yaml.MappingNode {
return root, nil
}
if root.Kind == yaml.SequenceNode {
for _, candidate := range root.Content {
if candidate.Kind == yaml.MappingNode {
name := mappingValue(candidate, "name")
if name != nil && name.Value == modelName {
return candidate, nil
}
}
}
}
return nil, fmt.Errorf("model %q not found in configuration document", modelName)
}
func mappingValue(mapping *yaml.Node, key string) *yaml.Node {
for index := 0; index+1 < len(mapping.Content); index += 2 {
if mapping.Content[index].Value == key {
return mapping.Content[index+1]
}
}
return nil
}
func setMappingValue(mapping *yaml.Node, key string, value *yaml.Node) {
for index := 0; index+1 < len(mapping.Content); index += 2 {
if mapping.Content[index].Value == key {
mapping.Content[index+1] = value
return
}
}
mapping.Content = append(mapping.Content,
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, value,
)
}
func writeBindingAtomic(fileName string, data []byte) error {
temporary, err := os.CreateTemp(filepath.Dir(fileName), ".artifact-binding-*")
if err != nil {
return err
}
temporaryName := temporary.Name()
defer func() { _ = os.Remove(temporaryName) }()
if err := temporary.Chmod(0600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(data); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
if err := os.Chmod(temporaryName, 0644); err != nil {
return err
}
return os.Rename(temporaryName, fileName)
}

View File

@@ -0,0 +1,50 @@
package config
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
var _ = Describe("artifact binding persistence", func() {
It("updates only the named model in a multi-config document", func() {
fileName := filepath.Join(GinkgoT().TempDir(), "models.yaml")
Expect(os.WriteFile(fileName, []byte(`
- name: managed
backend: transformers
unknown_extension: keep-me
artifacts:
- source: {type: huggingface, repo: owner/repo}
parameters: {model: owner/repo}
- name: sibling
backend: llama-cpp
sibling_only: true
parameters: {model: sibling.gguf}
`), 0644)).To(Succeed())
result := modelartifacts.Result{
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
Spec: modelartifacts.Spec{
Name: "model", Target: "model",
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
},
},
}
Expect(persistArtifactBinding(fileName, "managed", result)).To(Succeed())
updated, err := os.ReadFile(fileName)
Expect(err).NotTo(HaveOccurred())
Expect(string(updated)).To(ContainSubstring("name: sibling"))
Expect(string(updated)).To(ContainSubstring("sibling_only: true"))
Expect(string(updated)).To(ContainSubstring("unknown_extension: keep-me"))
Expect(string(updated)).To(ContainSubstring("model: owner/repo"))
Expect(string(updated)).To(ContainSubstring("cache_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
Expect(string(updated)).To(ContainSubstring("revision: 0123456789abcdef0123456789abcdef01234567"))
})
})

View File

@@ -0,0 +1,61 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
// PrimaryArtifactSpec returns the managed primary artifact to materialize for
// this config. The boolean return is false when the config should stay on the
// legacy path.
func (c ModelConfig) PrimaryArtifactSpec(modelsPath string) (modelartifacts.Spec, bool, bool, error) {
if len(c.Artifacts) > 0 {
return c.Artifacts[0], false, true, nil
}
modelRef := strings.TrimSpace(c.Model)
if modelRef == "" {
return modelartifacts.Spec{}, false, false, nil
}
if len(c.DownloadFiles) > 0 {
return modelartifacts.Spec{}, false, false, nil
}
if modelsPath != "" {
for _, candidate := range []string{
modelRef,
filepath.Join(modelsPath, modelRef),
} {
if info, err := os.Stat(candidate); err == nil && (info.IsDir() || info.Mode().IsRegular()) {
return modelartifacts.Spec{}, false, false, nil
}
}
}
spec, ok, err := modelartifacts.ParsePrimaryReference(modelRef)
if err != nil {
return modelartifacts.Spec{}, false, false, err
}
if !ok {
if strings.Contains(modelRef, "/") && !strings.HasPrefix(modelRef, ".") && !filepath.IsAbs(modelRef) {
repo, err := modelartifacts.CanonicalRepo(modelRef)
if err != nil {
if strings.Count(modelRef, "/") == 1 {
return modelartifacts.Spec{}, false, false, fmt.Errorf("invalid Hugging Face reference %q: %w", modelRef, err)
}
return modelartifacts.Spec{}, false, false, nil
}
return modelartifacts.Spec{
Name: modelartifacts.TargetModel,
Target: modelartifacts.TargetModel,
Source: modelartifacts.Source{Type: modelartifacts.SourceTypeHuggingFace, Repo: repo},
}, true, true, nil
}
return modelartifacts.Spec{}, false, false, nil
}
return spec, true, true, nil
}

View File

@@ -13,6 +13,7 @@ import (
"github.com/mudler/LocalAI/core/services/routing/piipattern"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/functions"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/reasoning"
"github.com/mudler/cogito"
"gopkg.in/yaml.v3"
@@ -40,7 +41,8 @@ type ModelConfig struct {
modelConfigFile string `yaml:"-" json:"-"`
modelTemplate string `yaml:"-" json:"-"`
schema.PredictionOptions `yaml:"parameters,omitempty" json:"parameters,omitempty"`
Name string `yaml:"name,omitempty" json:"name,omitempty"`
Name string `yaml:"name,omitempty" json:"name,omitempty"`
Artifacts []modelartifacts.Spec `yaml:"artifacts,omitempty" json:"artifacts,omitempty"`
// Alias, when set, makes this config a pure redirect: every request for
// Name is served by the model named here. All other fields are ignored.
@@ -1210,9 +1212,15 @@ func (c ModelConfig) ModelID() string {
return c.Model
}
// ModelFileName returns the filename of the model
// If the model is a URL, it will return the MD5 of the URL which is the filename
// ModelFileName returns the controller-managed snapshot when the model has a
// committed artifact, otherwise preserving the legacy URL/repository behavior.
func (c *ModelConfig) ModelFileName() string {
if len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil {
relative, err := modelartifacts.RelativeSnapshotPath(c.Artifacts[0].Resolved.CacheKey)
if err == nil {
return relative
}
}
uri := downloader.URI(c.Model)
if uri.LooksLikeURL() {
f, _ := uri.FilenameFromUrl()
@@ -1315,6 +1323,21 @@ func (cfg *ModelConfig) SetDefaults(opts ...ConfigLoaderOption) {
}
func (c *ModelConfig) Validate() (bool, error) {
if c.IsAlias() && len(c.Artifacts) > 0 {
return false, fmt.Errorf("alias model %q cannot declare artifacts", c.Name)
}
seenArtifacts := make(map[string]struct{}, len(c.Artifacts))
for i, artifact := range c.Artifacts {
normalized, err := artifact.Normalize()
if err != nil {
return false, fmt.Errorf("artifact %d: %w", i, err)
}
if _, exists := seenArtifacts[normalized.Name]; exists {
return false, fmt.Errorf("duplicate artifact name %q", normalized.Name)
}
seenArtifacts[normalized.Name] = struct{}{}
}
// An alias is a pure redirect: validate only its own shape here. Target
// existence and the no-chain rule need the full config set, so the loader
// (load-time) and the create/swap endpoints enforce those.

View File

@@ -2,11 +2,13 @@ package config
import (
"cmp"
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"sync"
@@ -14,22 +16,59 @@ import (
"github.com/charmbracelet/glamour"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/mudler/xlog"
"gopkg.in/yaml.v3"
)
// ArtifactMaterializer resolves and commits a model artifact into local storage.
type ArtifactMaterializer interface {
Ensure(context.Context, string, modelartifacts.Spec) (modelartifacts.Result, error)
}
// ModelConfigLoaderOption customizes a ModelConfigLoader at construction time.
type ModelConfigLoaderOption func(*ModelConfigLoader)
// WithArtifactMaterializer sets the controller-side artifact acquisition boundary.
func WithArtifactMaterializer(materializer ArtifactMaterializer) ModelConfigLoaderOption {
return func(loader *ModelConfigLoader) {
if materializer != nil {
loader.artifactMaterializer = materializer
}
}
}
// WithPreloadDisplay configures terminal rendering without reading process state.
func WithPreloadDisplay(renderMode string, disableColor bool) ModelConfigLoaderOption {
return func(loader *ModelConfigLoader) {
if renderMode != "" {
loader.preloadRenderMode = renderMode
}
loader.disablePreloadColor = disableColor
}
}
type ModelConfigLoader struct {
configs map[string]ModelConfig
modelPath string
configs map[string]ModelConfig
modelPath string
artifactMaterializer ArtifactMaterializer
preloadRenderMode string
disablePreloadColor bool
sync.Mutex
}
func NewModelConfigLoader(modelPath string) *ModelConfigLoader {
return &ModelConfigLoader{
configs: make(map[string]ModelConfig),
modelPath: modelPath,
func NewModelConfigLoader(modelPath string, options ...ModelConfigLoaderOption) *ModelConfigLoader {
loader := &ModelConfigLoader{
configs: make(map[string]ModelConfig),
modelPath: modelPath,
artifactMaterializer: modelartifacts.NewDefaultManager(),
preloadRenderMode: "dark",
}
for _, option := range options {
option(loader)
}
return loader
}
type LoadOptions struct {
@@ -351,98 +390,170 @@ func (bcl *ModelConfigLoader) ValidateAliasTarget(cfg *ModelConfig) error {
return nil
}
// Preload prepare models if they are not local but url or huggingface repositories
type preloadWork struct {
key string
config ModelConfig
}
// Preload prepares models if they are not local but URLs or Hugging Face repositories.
func (bcl *ModelConfigLoader) Preload(modelPath string) error {
return bcl.PreloadWithContext(context.Background(), modelPath)
}
// PreloadWithContext prepares remote model inputs while honoring cancellation.
func (bcl *ModelConfigLoader) PreloadWithContext(ctx context.Context, modelPath string) error {
bcl.Lock()
defer bcl.Unlock()
work := make([]preloadWork, 0, len(bcl.configs))
for key, config := range bcl.configs {
configCopy := config
configCopy.Artifacts = slices.Clone(config.Artifacts)
configCopy.DownloadFiles = slices.Clone(config.DownloadFiles)
work = append(work, preloadWork{key: key, config: configCopy})
}
bcl.Unlock()
status := func(fileName, current, total string, percent float64) {
utils.DisplayDownloadFunction(fileName, current, total, percent)
}
xlog.Info("Preloading models", "path", modelPath)
for _, item := range work {
if err := ctx.Err(); err != nil {
return err
}
updated, artifactResult, err := bcl.preloadOne(ctx, modelPath, item.config, status)
if err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
renderMode := "dark"
if os.Getenv("COLOR") != "" {
renderMode = os.Getenv("COLOR")
bcl.Lock()
current, exists := bcl.configs[item.key]
if !exists || !reflect.DeepEqual(current, item.config) {
bcl.Unlock()
continue
}
if artifactResult != nil && bindingNeedsPersistence(current, *artifactResult) && current.modelConfigFile != "" {
modelartifacts.ReportProgress(ctx, modelartifacts.ProgressEvent{
Phase: modelartifacts.PhasePersisting,
Artifact: artifactResult.Spec.Name,
})
if err := persistArtifactBinding(current.modelConfigFile, current.Name, *artifactResult); err != nil {
bcl.Unlock()
return err
}
}
bcl.configs[item.key] = updated
bcl.Unlock()
bcl.displayPreloadedModel(updated)
}
return nil
}
func (bcl *ModelConfigLoader) preloadOne(
ctx context.Context,
modelPath string,
config ModelConfig,
status func(string, string, string, float64),
) (ModelConfig, *modelartifacts.Result, error) {
updated := config
updated.Artifacts = slices.Clone(config.Artifacts)
tasks := make([]downloader.FileTask, 0, len(updated.DownloadFiles))
for index, file := range updated.DownloadFiles {
if err := ctx.Err(); err != nil {
return ModelConfig{}, nil, err
}
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
return ModelConfig{}, nil, err
}
tasks = append(tasks, downloader.FileTask{
URI: file.URI,
Destination: filepath.Join(modelPath, file.Filename),
SHA256: file.SHA256,
FileIndex: index,
TotalFiles: len(updated.DownloadFiles),
})
}
if err := downloader.DownloadFilesWithContext(ctx, tasks, status); err != nil {
return ModelConfig{}, nil, err
}
artifactSpec, inferred, managedPrimary, err := updated.PrimaryArtifactSpec(modelPath)
if err != nil {
return ModelConfig{}, nil, err
}
var artifactResult *modelartifacts.Result
if managedPrimary {
result, err := bcl.artifactMaterializer.Ensure(ctx, modelPath, artifactSpec)
if err != nil {
if inferred {
xlog.Warn("falling back to legacy model loading after artifact materialization failed", "model", updated.Name, "error", err)
managedPrimary = false
} else {
return ModelConfig{}, nil, err
}
} else {
next := []modelartifacts.Spec{result.Spec}
if len(updated.Artifacts) > 1 {
next = append(next, updated.Artifacts[1:]...)
}
updated.Artifacts = next
artifactResult = &result
}
}
if !managedPrimary && updated.IsModelURL() {
modelFileName := updated.ModelFileName()
uri := downloader.URI(updated.Model)
if uri.ResolveURL() != updated.Model {
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
if err := uri.DownloadFileWithContext(ctx, filepath.Join(modelPath, modelFileName), "", 0, 0, status); err != nil {
return ModelConfig{}, nil, err
}
}
updated.Model = modelFileName
}
}
if updated.IsMMProjURL() {
modelFileName := updated.MMProjFileName()
uri := downloader.URI(updated.MMProj)
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
if err := uri.DownloadFileWithContext(ctx, filepath.Join(modelPath, modelFileName), "", 0, 0, status); err != nil {
return ModelConfig{}, nil, err
}
}
updated.MMProj = modelFileName
}
return updated, artifactResult, nil
}
func bindingNeedsPersistence(current ModelConfig, result modelartifacts.Result) bool {
return len(current.Artifacts) == 0 || !reflect.DeepEqual(current.Artifacts[0], result.Spec)
}
func (bcl *ModelConfigLoader) displayPreloadedModel(config ModelConfig) {
glamText := func(t string) {
out, err := glamour.Render(t, renderMode)
if err == nil && os.Getenv("NO_COLOR") == "" {
out, err := glamour.Render(t, bcl.preloadRenderMode)
if err == nil && !bcl.disablePreloadColor {
fmt.Println(out)
} else {
fmt.Println(t)
}
}
for i, config := range bcl.configs {
// Download files and verify their SHA
for i, file := range config.DownloadFiles {
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
return err
}
// Create file path
filePath := filepath.Join(modelPath, file.Filename)
if err := file.URI.DownloadFile(filePath, file.SHA256, i, len(config.DownloadFiles), status); err != nil {
return err
}
}
// If the model is an URL, expand it, and download the file
if config.IsModelURL() {
modelFileName := config.ModelFileName()
uri := downloader.URI(config.Model)
if uri.ResolveURL() != config.Model {
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil {
return err
}
}
cc := bcl.configs[i]
c := &cc
c.PredictionOptions.Model = modelFileName
bcl.configs[i] = *c
}
}
if config.IsMMProjURL() {
modelFileName := config.MMProjFileName()
uri := downloader.URI(config.MMProj)
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil {
return err
}
}
cc := bcl.configs[i]
c := &cc
c.MMProj = modelFileName
bcl.configs[i] = *c
}
if bcl.configs[i].Name != "" {
glamText(fmt.Sprintf("**Model name**: _%s_", bcl.configs[i].Name))
}
if bcl.configs[i].Description != "" {
//glamText("**Description**")
glamText(bcl.configs[i].Description)
}
if bcl.configs[i].Usage != "" {
//glamText("**Usage**")
glamText(bcl.configs[i].Usage)
}
if config.Name != "" {
glamText(fmt.Sprintf("**Model name**: _%s_", config.Name))
}
if config.Description != "" {
glamText(config.Description)
}
if config.Usage != "" {
glamText(config.Usage)
}
return nil
}
// MITMHostOwnership is the result of mapping intercept hosts to the

View File

@@ -1,10 +1,215 @@
package config
import (
"context"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
type preloadArtifactMaterializer struct {
result modelartifacts.Result
err error
seen chan modelartifacts.Spec
release <-chan struct{}
}
func (f *preloadArtifactMaterializer) Ensure(ctx context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
if f.seen != nil {
f.seen <- spec
}
if f.release != nil {
select {
case <-ctx.Done():
return modelartifacts.Result{}, ctx.Err()
case <-f.release:
}
}
return f.result, f.err
}
var _ = Describe("ModelConfigLoader artifact preload", func() {
It("materializes and persists a source-only artifact binding", func() {
modelsPath := GinkgoT().TempDir()
configPath := filepath.Join(modelsPath, "managed.yaml")
Expect(os.WriteFile(configPath, []byte(`
name: managed
backend: transformers
unknown_extension: keep-me
artifacts:
- name: model
target: model
source: {type: huggingface, repo: owner/repo}
parameters: {model: owner/repo}
`), 0644)).To(Succeed())
resolved := modelartifacts.Spec{
Name: "model", Target: "model",
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
},
}
fake := &preloadArtifactMaterializer{result: modelartifacts.Result{
Spec: resolved,
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
}, seen: make(chan modelartifacts.Spec, 1)}
loader := NewModelConfigLoader(modelsPath, WithArtifactMaterializer(fake))
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
Expect(loader.PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
loaded, found := loader.GetModelConfig("managed")
Expect(found).To(BeTrue())
Expect(loaded.Model).To(Equal("owner/repo"))
Expect(loaded.ModelFileName()).To(Equal(fake.result.RelativePath))
data, err := os.ReadFile(configPath)
Expect(err).NotTo(HaveOccurred())
Expect(string(data)).To(ContainSubstring("unknown_extension: keep-me"))
Expect(string(data)).To(ContainSubstring("revision: 0123456789abcdef0123456789abcdef01234567"))
Expect(string(data)).To(ContainSubstring("cache_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
Expect(string(data)).To(ContainSubstring("model: owner/repo"))
})
It("materializes a direct Hugging Face file reference", func() {
modelsPath := GinkgoT().TempDir()
configPath := filepath.Join(modelsPath, "hf-file.yaml")
Expect(os.WriteFile(configPath, []byte(`
name: hf-file
backend: transformers
parameters:
model: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf
`), 0644)).To(Succeed())
resolved := modelartifacts.Spec{
Name: "model",
Target: "model",
Source: modelartifacts.Source{Type: "huggingface", Repo: "nomic-ai/nomic-embed-text-v1.5-GGUF", AllowPatterns: []string{"nomic-embed-text-v1.5.f16.gguf"}, Revision: "main"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
},
}
fake := &preloadArtifactMaterializer{result: modelartifacts.Result{
Spec: resolved,
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
}, seen: make(chan modelartifacts.Spec, 1)}
loader := NewModelConfigLoader(modelsPath, WithArtifactMaterializer(fake))
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
Expect(loader.PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
loaded, found := loader.GetModelConfig("hf-file")
Expect(found).To(BeTrue())
Expect(loaded.Model).To(Equal("https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf"))
Expect(loaded.ModelFileName()).To(Equal(fake.result.RelativePath))
Expect(fake.seen).To(Receive(SatisfyAll(
WithTransform(func(spec modelartifacts.Spec) string { return spec.Source.Repo }, Equal("nomic-ai/nomic-embed-text-v1.5-GGUF")),
WithTransform(func(spec modelartifacts.Spec) []string { return spec.Source.AllowPatterns }, Equal([]string{"nomic-embed-text-v1.5.f16.gguf"})),
)))
})
It("falls back to the legacy path when inferred materialization fails", func() {
modelsPath := GinkgoT().TempDir()
configPath := filepath.Join(modelsPath, "hf-legacy.yaml")
Expect(os.WriteFile(configPath, []byte(`
name: hf-legacy
backend: transformers
parameters:
model: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf
`), 0644)).To(Succeed())
fake := &preloadArtifactMaterializer{err: context.Canceled, seen: make(chan modelartifacts.Spec, 1)}
loader := NewModelConfigLoader(modelsPath, WithArtifactMaterializer(fake))
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
Expect(loader.PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
loaded, found := loader.GetModelConfig("hf-legacy")
Expect(found).To(BeTrue())
Expect(loaded.Artifacts).To(BeEmpty())
Expect(loaded.Model).To(Equal("https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf"))
Expect(fake.seen).To(Receive())
})
It("does not hold the loader lock while materialization blocks", func() {
seen := make(chan modelartifacts.Spec, 1)
release := make(chan struct{})
fake := &preloadArtifactMaterializer{seen: seen, release: release}
loader := NewModelConfigLoader(GinkgoT().TempDir(), WithArtifactMaterializer(fake))
loader.Lock()
loader.configs["managed"] = ModelConfig{
Name: "managed",
Artifacts: []modelartifacts.Spec{{
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
}},
}
loader.Unlock()
done := make(chan error, 1)
go func() { done <- loader.PreloadWithContext(context.Background(), loader.modelPath) }()
<-seen
lookupDone := make(chan struct{})
go func() {
_, _ = loader.GetModelConfig("managed")
close(lookupDone)
}()
Eventually(lookupDone).Should(BeClosed())
close(release)
Expect(<-done).NotTo(HaveOccurred())
})
It("propagates preload cancellation", func() {
seen := make(chan modelartifacts.Spec, 1)
release := make(chan struct{})
fake := &preloadArtifactMaterializer{seen: seen, release: release}
loader := NewModelConfigLoader(GinkgoT().TempDir(), WithArtifactMaterializer(fake))
loader.Lock()
loader.configs["managed"] = ModelConfig{
Name: "managed",
Artifacts: []modelartifacts.Spec{{
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
}},
}
loader.Unlock()
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- loader.PreloadWithContext(ctx, loader.modelPath) }()
<-seen
cancel()
Expect(<-done).To(MatchError(context.Canceled))
})
It("does not overwrite a config changed during materialization", func() {
seen := make(chan modelartifacts.Spec, 1)
release := make(chan struct{})
fake := &preloadArtifactMaterializer{
seen: seen, release: release,
result: modelartifacts.Result{RelativePath: ".artifacts/huggingface/cached/snapshot"},
}
loader := NewModelConfigLoader(GinkgoT().TempDir(), WithArtifactMaterializer(fake))
loader.Lock()
loader.configs["managed"] = ModelConfig{
Name: "managed", Description: "before",
Artifacts: []modelartifacts.Spec{{
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
}},
}
loader.Unlock()
done := make(chan error, 1)
go func() { done <- loader.PreloadWithContext(context.Background(), loader.modelPath) }()
<-seen
loader.UpdateModelConfig("managed", func(cfg *ModelConfig) {
cfg.Description = "changed concurrently"
})
close(release)
Expect(<-done).NotTo(HaveOccurred())
loaded, found := loader.GetModelConfig("managed")
Expect(found).To(BeTrue())
Expect(loaded.Description).To(Equal("changed concurrently"))
})
})
var _ = Describe("ModelConfigLoader.GetModelsConflictingWith", func() {
var bcl *ModelConfigLoader

View File

@@ -4,9 +4,12 @@ import (
"io"
"net/http"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"gopkg.in/yaml.v3"
)
@@ -28,6 +31,40 @@ var _ = Describe("Test cases for config related functions", func() {
})
})
It("round-trips and validates a managed model artifact", func() {
raw := []byte(`
name: qwen-asr
backend: qwen-asr
artifacts:
- name: model
target: model
source:
type: huggingface
repo: Qwen/Qwen3-ASR-1.7B
parameters:
model: Qwen/Qwen3-ASR-1.7B
`)
var cfg ModelConfig
Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed())
Expect(cfg.Artifacts).To(HaveLen(1))
valid, err := cfg.Validate()
Expect(err).NotTo(HaveOccurred())
Expect(valid).To(BeTrue())
})
It("derives a managed snapshot filename without replacing the logical model", func() {
const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
cfg := ModelConfig{
Artifacts: []modelartifacts.Spec{{
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
Resolved: &modelartifacts.Resolved{CacheKey: cacheKey},
}},
}
cfg.Model = "owner/repo"
Expect(cfg.Model).To(Equal("owner/repo"))
Expect(cfg.ModelFileName()).To(Equal(filepath.Join(".artifacts", "huggingface", cacheKey, "snapshot")))
})
Context("Test Read configuration functions", func() {
It("Test Validate", func() {
tmp, err := os.CreateTemp("", "config.yaml")
@@ -834,4 +871,17 @@ var _ = Describe("ModelConfig alias", func() {
Expect(ok).To(BeFalse())
Expect(err).To(MatchError(ContainSubstring("pure redirect")))
})
It("rejects artifacts on alias configurations", func() {
cfg := ModelConfig{
Name: "alias-name",
Alias: "target-name",
Artifacts: []modelartifacts.Spec{{
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
}},
}
valid, err := cfg.Validate()
Expect(valid).To(BeFalse())
Expect(err).To(MatchError(ContainSubstring("alias")))
})
})

View File

@@ -4,10 +4,57 @@ import (
"path/filepath"
"strings"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/downloader"
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"gopkg.in/yaml.v3"
)
var managedArtifactBackends = map[string]struct{}{
"transformers": {}, "huggingface-embeddings": {}, "sentencetransformers": {},
"transformers-musicgen": {}, "mamba": {}, "diffusers": {}, "qwen-asr": {},
"fish-speech": {}, "nemo": {}, "voxcpm": {}, "qwen-tts": {},
"liquid-audio": {}, "vllm": {}, "vllm-omni": {}, "sglang": {},
}
// AttachPrimaryArtifact adds the controller-managed source only when the
// importer selected the same repository and a migrated backend.
func AttachPrimaryArtifact(model gallery.ModelConfig, details Details) (gallery.ModelConfig, error) {
if len(model.Files) != 0 || details.HuggingFace == nil || details.HuggingFace.ModelID == "" {
return model, nil
}
var cfg config.ModelConfig
if err := yaml.Unmarshal([]byte(model.ConfigFile), &cfg); err != nil {
return gallery.ModelConfig{}, err
}
if _, supported := managedArtifactBackends[cfg.Backend]; !supported {
return model, nil
}
if len(cfg.Artifacts) != 0 || cfg.Model != details.HuggingFace.ModelID {
return model, nil
}
var document map[string]any
if err := yaml.Unmarshal([]byte(model.ConfigFile), &document); err != nil {
return gallery.ModelConfig{}, err
}
document["artifacts"] = []map[string]any{{
"name": modelartifacts.TargetModel,
"target": modelartifacts.TargetModel,
"source": map[string]any{
"type": modelartifacts.SourceTypeHuggingFace,
"repo": details.HuggingFace.ModelID,
},
}}
encoded, err := yaml.Marshal(document)
if err != nil {
return gallery.ModelConfig{}, err
}
model.ConfigFile = string(encoded)
return model, nil
}
// LocalModelPath normalizes a model URI for backends that treat the model
// field as a HuggingFace repo id or local filesystem path (mlx, mlx-vlm,
// vllm, transformers, diffusers). A "file://" import URI is reduced to the

View File

@@ -4,8 +4,11 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/gallery/importers"
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
"gopkg.in/yaml.v3"
)
var _ = Describe("importer helpers", func() {
@@ -104,3 +107,50 @@ var _ = Describe("importer helpers", func() {
})
})
})
var _ = Describe("managed artifact attachment", func() {
details := importers.Details{
URI: "https://huggingface.co/owner/repo",
HuggingFace: &hfapi.ModelDetails{ModelID: "owner/repo"},
}
It("attaches a source when the imported model is the discovered repository", func() {
input := gallery.ModelConfig{ConfigFile: "backend: transformers\nparameters:\n model: owner/repo\n"}
output, err := importers.AttachPrimaryArtifact(input, details)
Expect(err).NotTo(HaveOccurred())
var cfg config.ModelConfig
Expect(yaml.Unmarshal([]byte(output.ConfigFile), &cfg)).To(Succeed())
Expect(cfg.Artifacts).To(HaveLen(1))
Expect(cfg.Artifacts[0].Source.Repo).To(Equal("owner/repo"))
})
It("preserves explicit gallery files", func() {
input := gallery.ModelConfig{
ConfigFile: "parameters:\n model: owner/repo\n",
Files: []gallery.File{{Filename: "model.gguf", URI: "https://huggingface.co/owner/repo/resolve/main/model.gguf"}},
}
output, err := importers.AttachPrimaryArtifact(input, details)
Expect(err).NotTo(HaveOccurred())
Expect(output.ConfigFile).To(Equal(input.ConfigFile))
})
It("does not attach a managed source to an unmigrated backend", func() {
input := gallery.ModelConfig{ConfigFile: "backend: custom-python\nparameters:\n model: owner/repo\n"}
output, err := importers.AttachPrimaryArtifact(input, details)
Expect(err).NotTo(HaveOccurred())
Expect(output.ConfigFile).To(Equal(input.ConfigFile))
})
DescribeTable("does not guess an unrelated or local source",
func(model string, candidate importers.Details) {
input := gallery.ModelConfig{ConfigFile: "parameters:\n model: " + model + "\n"}
output, err := importers.AttachPrimaryArtifact(input, candidate)
Expect(err).NotTo(HaveOccurred())
var cfg config.ModelConfig
Expect(yaml.Unmarshal([]byte(output.ConfigFile), &cfg)).To(Succeed())
Expect(cfg.Artifacts).To(BeEmpty())
},
Entry("different repo", "other/repo", details),
Entry("local file", "/models/repo", importers.Details{URI: "file:///models/repo"}),
)
})

View File

@@ -318,6 +318,10 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
if err != nil {
continue
}
modelConfig, err = AttachPrimaryArtifact(modelConfig, details)
if err != nil {
return gallery.ModelConfig{}, fmt.Errorf("attach managed artifact: %w", err)
}
break
}
}

View File

@@ -0,0 +1,94 @@
package gallery
import (
"context"
"fmt"
"os"
"path/filepath"
"gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/xlog"
)
type ArtifactMaterializer interface {
Ensure(context.Context, string, modelartifacts.Spec) (modelartifacts.Result, error)
}
type installOptions struct {
materializer ArtifactMaterializer
}
type InstallOption func(*installOptions)
func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption {
return func(options *installOptions) {
if materializer != nil {
options.materializer = materializer
}
}
}
func applyInstallOptions(options ...InstallOption) installOptions {
result := installOptions{materializer: modelartifacts.NewDefaultManager()}
for _, option := range options {
option(&result)
}
return result
}
func bindPrimaryArtifact(ctx context.Context, modelsPath string, typed *config.ModelConfig, configMap map[string]any, materializer ArtifactMaterializer, artifactSpec modelartifacts.Spec, inferred bool) (bool, error) {
result, err := materializer.Ensure(ctx, modelsPath, artifactSpec)
if err != nil {
if inferred {
xlog.Warn("falling back to legacy model loading after artifact materialization failed", "model", typed.Name, "error", err)
return false, nil
}
return false, fmt.Errorf("materialize primary model artifact: %w", err)
}
next := []modelartifacts.Spec{result.Spec}
if len(typed.Artifacts) > 1 {
next = append(next, typed.Artifacts[1:]...)
}
typed.Artifacts = next
artifactYAML, err := yaml.Marshal(typed.Artifacts)
if err != nil {
return false, err
}
var artifactValue any
if err := yaml.Unmarshal(artifactYAML, &artifactValue); err != nil {
return false, err
}
configMap["artifacts"] = artifactValue
return true, nil
}
func writeModelConfigAtomic(fileName string, data []byte) error {
temporary, err := os.CreateTemp(filepath.Dir(fileName), ".model-config-*")
if err != nil {
return err
}
temporaryName := temporary.Name()
defer func() { _ = os.Remove(temporaryName) }()
if err := temporary.Chmod(0600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(data); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
if err := os.Chmod(temporaryName, 0644); err != nil {
return err
}
return os.Rename(temporaryName, fileName)
}

View File

@@ -0,0 +1,240 @@
package gallery_test
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
)
type fakeArtifactMaterializer struct {
result modelartifacts.Result
err error
seen []modelartifacts.Spec
}
func (f *fakeArtifactMaterializer) Ensure(_ context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
f.seen = append(f.seen, spec)
return f.result, f.err
}
var _ = Describe("gallery artifact installation", func() {
It("persists resolved state only after materialization succeeds", func() {
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).NotTo(HaveOccurred())
resolved := modelartifacts.Spec{
Name: "model", Target: "model",
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
},
}
fake := &fakeArtifactMaterializer{result: modelartifacts.Result{
Spec: resolved,
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
}}
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
backend: transformers
artifacts:
- name: model
target: model
source:
type: huggingface
repo: owner/repo
parameters:
model: owner/repo
unknown_extension:
keep: true
`}
installed, err := gallery.InstallModel(context.Background(), state, "managed", definition, nil, nil, false,
gallery.WithArtifactMaterializer(fake))
Expect(err).NotTo(HaveOccurred())
Expect(fake.seen).To(HaveLen(1))
Expect(installed.Model).To(Equal("owner/repo"))
Expect(installed.ModelFileName()).To(Equal(fake.result.RelativePath))
data, err := os.ReadFile(filepath.Join(modelsPath, "managed.yaml"))
Expect(err).NotTo(HaveOccurred())
var persisted map[string]any
Expect(yaml.Unmarshal(data, &persisted)).To(Succeed())
Expect(persisted).To(HaveKey("unknown_extension"))
parameters := persisted["parameters"].(map[string]any)
Expect(parameters["model"]).To(Equal("owner/repo"))
artifacts := persisted["artifacts"].([]any)
resolvedMap := artifacts[0].(map[string]any)["resolved"].(map[string]any)
Expect(resolvedMap["revision"]).To(Equal(resolved.Resolved.Revision))
})
It("does not replace an existing config when materialization fails", func() {
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).NotTo(HaveOccurred())
configPath := filepath.Join(modelsPath, "managed.yaml")
Expect(os.WriteFile(configPath, []byte("name: old\n"), 0644)).To(Succeed())
fake := &fakeArtifactMaterializer{err: errors.New("acquisition failed")}
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
artifacts:
- source: {type: huggingface, repo: owner/repo}
parameters: {model: owner/repo}
`}
_, err = gallery.InstallModel(context.Background(), state, "managed", definition, nil, nil, false,
gallery.WithArtifactMaterializer(fake))
Expect(err).To(MatchError(ContainSubstring("acquisition failed")))
Expect(os.ReadFile(configPath)).To(Equal([]byte("name: old\n")))
})
It("does not replace an existing config when a legacy file download fails", func() {
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).NotTo(HaveOccurred())
configPath := filepath.Join(modelsPath, "legacy.yaml")
Expect(os.WriteFile(configPath, []byte("name: old\n"), 0644)).To(Succeed())
definition := &gallery.ModelConfig{
Name: "legacy",
ConfigFile: "parameters: {model: owner/legacy}\n",
Files: []gallery.File{{
Filename: "missing.bin",
URI: "file:///definitely-not-a-localai-test-file",
}},
}
_, err = gallery.InstallModel(context.Background(), state, "legacy", definition, nil, nil, false)
Expect(err).To(HaveOccurred())
Expect(os.ReadFile(configPath)).To(Equal([]byte("name: old\n")))
})
It("blocks an explicitly unsafe repository before materialization", func() {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expect(r.URL.Path).To(Equal("/api/models/owner/repo/scan"))
w.Header().Set("Content-Type", "application/json")
_, err := w.Write([]byte(`{"hasUnsafeFile":true}`))
Expect(err).NotTo(HaveOccurred())
}))
DeferCleanup(server.Close)
originalEndpoint := downloader.HF_ENDPOINT
downloader.HF_ENDPOINT = server.URL
DeferCleanup(func() { downloader.HF_ENDPOINT = originalEndpoint })
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).NotTo(HaveOccurred())
fake := &fakeArtifactMaterializer{err: errors.New("unsafe install must not materialize")}
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
artifacts:
- source: {type: huggingface, repo: owner/repo}
parameters: {model: owner/repo}
`}
_, err = gallery.InstallModel(context.Background(), state, "managed", definition, nil, nil, true,
gallery.WithArtifactMaterializer(fake))
Expect(err).To(MatchError(downloader.ErrUnsafeFilesFound))
Expect(fake.seen).To(BeEmpty())
})
It("falls back to the legacy path when inferred materialization fails", func() {
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).NotTo(HaveOccurred())
fake := &fakeArtifactMaterializer{err: errors.New("legacy install must not materialize")}
definition := &gallery.ModelConfig{Name: "legacy", ConfigFile: `
backend: transformers
parameters:
model: owner/legacy
`}
installed, err := gallery.InstallModel(
context.Background(), state, "legacy", definition, nil, nil, false,
gallery.WithArtifactMaterializer(fake),
)
Expect(err).NotTo(HaveOccurred())
Expect(fake.seen).To(HaveLen(1))
Expect(installed.Model).To(Equal("owner/legacy"))
Expect(installed.Artifacts).To(BeEmpty())
Expect(filepath.Join(modelsPath, "legacy.yaml")).To(BeAnExistingFile())
})
It("passes the controller materializer through named gallery installs", func() {
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).NotTo(HaveOccurred())
definitionPath := filepath.Join(modelsPath, "definition.yaml")
definition, err := yaml.Marshal(gallery.ModelConfig{Name: "managed", ConfigFile: `
backend: transformers
artifacts:
- source: {type: huggingface, repo: owner/repo}
parameters: {model: owner/repo}
`})
Expect(err).NotTo(HaveOccurred())
Expect(os.WriteFile(definitionPath, definition, 0644)).To(Succeed())
galleryPath := filepath.Join(modelsPath, "index.yaml")
index, err := yaml.Marshal([]gallery.GalleryModel{{Metadata: gallery.Metadata{
Name: "managed",
URL: "file://" + definitionPath,
}}})
Expect(err).NotTo(HaveOccurred())
Expect(os.WriteFile(galleryPath, index, 0644)).To(Succeed())
galleries := []config.Gallery{{Name: "test", URL: "file://" + galleryPath}}
fake := &fakeArtifactMaterializer{result: modelartifacts.Result{Spec: modelartifacts.Spec{
Name: "model", Target: "model",
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
},
}}}
Expect(gallery.InstallModelFromGallery(
context.Background(), galleries, nil, state, nil, "test@managed",
gallery.GalleryModel{}, nil, false, false, false,
gallery.WithArtifactMaterializer(fake),
)).To(Succeed())
Expect(fake.seen).To(HaveLen(1))
})
})
var _ = Describe("artifact cache deletion policy", func() {
It("deletes the installed config without treating its snapshot directory as a regular file", func() {
const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
modelsPath := GinkgoT().TempDir()
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
Expect(err).NotTo(HaveOccurred())
artifactRoot := filepath.Join(modelsPath, ".artifacts", "huggingface", cacheKey)
Expect(os.MkdirAll(filepath.Join(artifactRoot, "snapshot"), 0750)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artifactRoot, "manifest.json"), []byte("{}"), 0644)).To(Succeed())
Expect(os.WriteFile(filepath.Join(modelsPath, "managed.yaml"), []byte(`
name: managed
artifacts:
- name: model
target: model
source: {type: huggingface, repo: owner/repo}
resolved:
endpoint: https://huggingface.co
revision: 0123456789abcdef0123456789abcdef01234567
cache_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
parameters:
model: owner/repo
`), 0644)).To(Succeed())
Expect(gallery.DeleteModelFromSystem(state, "managed")).To(Succeed())
Expect(filepath.Join(modelsPath, "managed.yaml")).NotTo(BeAnExistingFile())
Expect(artifactRoot).To(BeADirectory())
Expect(filepath.Join(artifactRoot, "snapshot")).To(BeADirectory())
Expect(filepath.Join(artifactRoot, "manifest.json")).To(BeAnExistingFile())
})
})

View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"slices"
@@ -13,6 +14,7 @@ import (
lconfig "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
@@ -77,7 +79,7 @@ func InstallModelFromGallery(
modelGalleries, backendGalleries []lconfig.Gallery,
systemState *system.SystemState,
modelLoader *model.ModelLoader,
name string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend, requireBackendIntegrity bool) error {
name string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend, requireBackendIntegrity bool, options ...InstallOption) error {
applyModel := func(model *GalleryModel) error {
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
@@ -129,7 +131,7 @@ func InstallModelFromGallery(
}
}
installedModel, err := InstallModel(ctx, systemState, installName, &config, model.Overrides, downloadStatus, enforceScan)
installedModel, err := InstallModel(ctx, systemState, installName, &config, model.Overrides, downloadStatus, enforceScan, options...)
if err != nil {
return err
}
@@ -158,7 +160,9 @@ func InstallModelFromGallery(
return applyModel(model)
}
func InstallModel(ctx context.Context, systemState *system.SystemState, nameOverride string, config *ModelConfig, configOverrides map[string]any, downloadStatus func(string, string, string, float64), enforceScan bool) (*lconfig.ModelConfig, error) {
func InstallModel(ctx context.Context, systemState *system.SystemState, nameOverride string, galleryConfig *ModelConfig, configOverrides map[string]any, downloadStatus func(string, string, string, float64), enforceScan bool, options ...InstallOption) (*lconfig.ModelConfig, error) {
installOptions := applyInstallOptions(options...)
config := galleryConfig
basePath := systemState.Model.ModelsPath
// Create base path if it doesn't exist
err := os.MkdirAll(basePath, 0750)
@@ -170,59 +174,6 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
xlog.Debug("Config overrides", "overrides", configOverrides)
}
// Download files and verify their SHA
for i, file := range config.Files {
// Check for cancellation before each file
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, basePath); err != nil {
return nil, err
}
// Create file path
filePath := filepath.Join(basePath, file.Filename)
if enforceScan {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
xlog.Error("Contains unsafe file(s)!", "model", config.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
return nil, err
}
}
uri := downloader.URI(file.URI)
if err := uri.DownloadFileWithContext(ctx, filePath, file.SHA256, i, len(config.Files), downloadStatus); err != nil {
return nil, err
}
}
// Write prompt template contents to separate files
for _, template := range config.PromptTemplates {
if err := utils.VerifyPath(template.Name+".tmpl", basePath); err != nil {
return nil, err
}
// Create file path
filePath := filepath.Join(basePath, template.Name+".tmpl")
// Create parent directory
err := os.MkdirAll(filepath.Dir(filePath), 0750)
if err != nil {
return nil, fmt.Errorf("failed to create parent directory for prompt template %q: %v", template.Name, err)
}
// Create and write file content
err = os.WriteFile(filePath, []byte(template.Content), 0644)
if err != nil {
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
}
xlog.Debug("Prompt template written", "template", template.Name)
}
name := config.Name
if nameOverride != "" {
name = nameOverride
@@ -233,11 +184,11 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
}
modelConfig := lconfig.ModelConfig{}
configFilePath := filepath.Join(basePath, name+".yaml")
var updatedConfigYAML []byte
writeConfig := len(configOverrides) != 0 || len(config.ConfigFile) != 0
// write config file
if len(configOverrides) != 0 || len(config.ConfigFile) != 0 {
configFilePath := filepath.Join(basePath, name+".yaml")
if writeConfig {
// Read and update config file as map[string]interface{}
configMap := make(map[string]any)
err = yaml.Unmarshal([]byte(config.ConfigFile), &configMap)
@@ -253,8 +204,8 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
}
}
// Write updated config file
updatedConfigYAML, err := yaml.Marshal(configMap)
// Marshal the merged map for typed validation and default application.
updatedConfigYAML, err = yaml.Marshal(configMap)
if err != nil {
return nil, fmt.Errorf("failed to marshal updated config YAML: %v", err)
}
@@ -301,19 +252,110 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
}
}
// Re-marshal from configMap to preserve unknown fields
updatedConfigYAML, err = yaml.Marshal(configMap)
if err != nil {
return nil, fmt.Errorf("failed to marshal config with inference defaults: %v", err)
}
if valid, err := modelConfig.Validate(); !valid {
return nil, fmt.Errorf("failed to validate updated config YAML: %v", err)
}
err = os.WriteFile(configFilePath, updatedConfigYAML, 0644)
if len(config.Files) == 0 {
artifactSpec, inferred, hasArtifact, err := modelConfig.PrimaryArtifactSpec(basePath)
if err != nil {
return nil, err
}
if hasArtifact && enforceScan {
artifact, err := artifactSpec.Normalize()
if err != nil {
return nil, err
}
probe := downloader.URI(fmt.Sprintf("%s/%s/resolve/%s/.gitattributes", strings.TrimRight(downloader.HF_ENDPOINT, "/"), artifact.Source.Repo, url.PathEscape(artifact.Source.Revision)))
if _, err := downloader.HuggingFaceScan(probe); errors.Is(err, downloader.ErrUnsafeFilesFound) {
return nil, err
}
}
if hasArtifact {
applied, err := bindPrimaryArtifact(ctx, basePath, &modelConfig, configMap, installOptions.materializer, artifactSpec, inferred)
if err != nil {
return nil, err
}
if applied {
// Re-marshal from configMap after artifact binding to preserve unknown fields.
updatedConfigYAML, err = yaml.Marshal(configMap)
if err != nil {
return nil, fmt.Errorf("failed to marshal config with inference defaults: %v", err)
}
}
}
}
}
// Download files and verify their SHA
tasks := make([]downloader.FileTask, 0, len(config.Files))
for i, file := range config.Files {
// Check for cancellation before each file
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, basePath); err != nil {
return nil, err
}
// Create file path
filePath := filepath.Join(basePath, file.Filename)
if enforceScan {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
xlog.Error("Contains unsafe file(s)!", "model", config.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
return nil, err
}
}
tasks = append(tasks, downloader.FileTask{
URI: downloader.URI(file.URI),
Destination: filePath,
SHA256: file.SHA256,
FileIndex: i,
TotalFiles: len(config.Files),
})
}
if err := downloader.DownloadFilesWithContext(ctx, tasks, downloadStatus); err != nil {
return nil, err
}
// Write prompt template contents to separate files
for _, template := range config.PromptTemplates {
if err := utils.VerifyPath(template.Name+".tmpl", basePath); err != nil {
return nil, err
}
// Create file path
filePath := filepath.Join(basePath, template.Name+".tmpl")
// Create parent directory
err := os.MkdirAll(filepath.Dir(filePath), 0750)
if err != nil {
return nil, fmt.Errorf("failed to write updated config file: %v", err)
return nil, fmt.Errorf("failed to create parent directory for prompt template %q: %v", template.Name, err)
}
// Create and write file content
err = os.WriteFile(filePath, []byte(template.Content), 0644)
if err != nil {
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
}
xlog.Debug("Prompt template written", "template", template.Name)
}
if writeConfig {
if len(modelConfig.Artifacts) > 0 {
modelartifacts.ReportProgress(ctx, modelartifacts.ProgressEvent{
Phase: modelartifacts.PhasePersisting, Artifact: modelConfig.Artifacts[0].Name,
})
}
if err := writeModelConfigAtomic(configFilePath, updatedConfigYAML); err != nil {
return nil, fmt.Errorf("failed to atomically write updated config file: %w", err)
}
xlog.Debug("Written config file", "file", configFilePath)
@@ -374,7 +416,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
if err != nil {
return nil, err
}
if modelConfig.Model != "" {
if modelConfig.Model != "" && len(modelConfig.Artifacts) == 0 {
additionalFiles = append(additionalFiles, modelConfig.ModelFileName())
}

View File

@@ -0,0 +1,40 @@
import { test, expect } from './coverage-fixtures.js'
test('operations bar shows managed model acquisition phase and bytes', async ({ page }) => {
await page.route('**/api/operations', (route) => route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
operations: [{
id: 'qwen-asr',
name: 'qwen-asr',
fullName: 'qwen-asr',
jobID: 'artifact-job-123',
progress: 45,
taskType: 'installation',
isDeletion: false,
isBackend: false,
isQueued: false,
isCancelled: false,
cancellable: true,
phase: 'downloading',
currentBytes: 1073741824,
totalBytes: 4294967296,
}],
}),
}))
let cancelledPath = ''
await page.route('**/api/operations/artifact-job-123/cancel', (route) => {
cancelledPath = new URL(route.request().url()).pathname
return route.fulfill({ contentType: 'application/json', body: '{}' })
})
await page.goto('/app/models')
const operation = page.locator('.operation-item').filter({ hasText: 'qwen-asr' })
await expect(operation).toContainText('Downloading model files')
await expect(operation).toContainText('1 GB / 4 GB')
await expect(operation.locator('.operation-progress')).toHaveText('45%')
await expect(operation.locator('.operation-bar')).toHaveAttribute('style', /width: 45%/)
await operation.getByTitle('Cancel').click()
expect(cancelledPath).toBe('/api/operations/artifact-job-123/cancel')
})

View File

@@ -1,5 +1,14 @@
import { useState } from 'react'
import { useOperations } from '../hooks/useOperations'
import { formatBytes } from '../utils/format'
const artifactPhaseLabels = {
resolving: 'Resolving model files',
downloading: 'Downloading model files',
verifying: 'Verifying model files',
committing: 'Finalizing model installation',
persisting: 'Saving model configuration',
}
const nodeStatusLabels = {
success: 'Done',
@@ -26,6 +35,10 @@ export default function OperationsBar() {
const nodes = Array.isArray(op.nodes) ? op.nodes : []
const canExpand = nodes.length > 1
const isOpen = !!expanded[key]
const phaseLabel = artifactPhaseLabels[op.phase]
const byteLabel = Number.isFinite(op.currentBytes) && Number.isFinite(op.totalBytes) && op.totalBytes > 0
? `${formatBytes(op.currentBytes)} / ${formatBytes(op.totalBytes)}`
: ''
return (
<div key={key} className="operation-item">
<div className="operation-info">
@@ -68,7 +81,17 @@ export default function OperationsBar() {
Cancelling...
</span>
)}
{!op.error && op.message && !op.isQueued && !op.isCancelled && (
{!op.error && phaseLabel && !op.isCancelled && (
<span className="operation-phase" style={{ fontSize: '0.75rem', color: 'var(--color-text-muted)', marginLeft: 'var(--spacing-xs)' }}>
{phaseLabel}
</span>
)}
{!op.error && byteLabel && !op.isCancelled && (
<span className="operation-bytes" style={{ fontSize: '0.75rem', color: 'var(--color-text-muted)', marginLeft: 'var(--spacing-xs)' }}>
{byteLabel}
</span>
)}
{!op.error && op.message && !phaseLabel && !op.isQueued && !op.isCancelled && (
<span style={{ fontSize: '0.75rem', color: 'var(--color-text-muted)', marginLeft: 'var(--spacing-xs)' }}>
{op.message}
</span>

View File

@@ -173,6 +173,9 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
isCancelled := false
isCancellable := false
message := ""
phase := ""
currentBytes := int64(0)
totalBytes := int64(0)
if status != nil {
// Skip successfully completed operations
@@ -189,6 +192,9 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
isCancelled = status.Cancelled
isCancellable = status.Cancellable
message = status.Message
phase = status.Phase
currentBytes = status.CurrentBytes
totalBytes = status.TotalBytes
if isDeletion {
taskType = "deletion"
}
@@ -257,6 +263,15 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
if scopedNodeID != "" {
opData["nodeID"] = scopedNodeID
}
if phase != "" {
opData["phase"] = phase
}
if currentBytes > 0 {
opData["currentBytes"] = currentBytes
}
if totalBytes > 0 {
opData["totalBytes"] = totalBytes
}
if status != nil && status.Error != nil {
opData["error"] = status.Error.Error()
}
@@ -909,7 +924,8 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
})
}
_, err = gallery.InstallModel(context.Background(), appConfig.SystemState, model.Name, &config, model.Overrides, nil, false)
_, err = gallery.InstallModel(context.Background(), appConfig.SystemState, model.Name, &config, model.Overrides, nil, false,
gallery.WithArtifactMaterializer(appConfig.ModelArtifactMaterializer))
if err != nil {
return c.JSON(http.StatusInternalServerError, map[string]any{
"error": err.Error(),

View File

@@ -13,6 +13,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/routes"
"github.com/mudler/LocalAI/core/services/galleryop"
"github.com/mudler/LocalAI/pkg/system"
)
// These specs guard the contract between the opcache (which stores
@@ -152,4 +153,41 @@ var _ = Describe("/api/operations with node-scoped backend ops", func() {
Expect(found).ToNot(HaveKey("nodeID"), "non-node-scoped ops must NOT carry a nodeID field")
Expect(found["name"]).To(Equal("llama-cpp"))
})
It("surfaces managed model artifact phase and byte counters", func() {
state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
Expect(err).NotTo(HaveOccurred())
appCfg := &config.ApplicationConfig{SystemState: state}
galleryService := galleryop.NewGalleryService(appCfg, nil)
opcache := galleryop.NewOpCache(galleryService)
jobID := "test-op-artifact-progress"
opcache.Set("qwen-asr", jobID)
galleryService.UpdateStatus(jobID, &galleryop.OpStatus{
Phase: "downloading", CurrentBytes: 1024, TotalBytes: 4096,
Progress: 22.5, Message: "Downloading model file: weights.safetensors", Cancellable: true,
})
e := echo.New()
routes.RegisterUIAPIRoutes(e, nil, nil, appCfg, galleryService, opcache, &application.Application{}, noopMw)
req := httptest.NewRequest(http.MethodGet, "/api/operations", nil)
rec := httptest.NewRecorder()
e.ServeHTTP(rec, req)
Expect(rec.Code).To(Equal(http.StatusOK))
var envelope struct {
Operations []map[string]any `json:"operations"`
}
Expect(json.Unmarshal(rec.Body.Bytes(), &envelope)).To(Succeed())
var found map[string]any
for _, op := range envelope.Operations {
if op["jobID"] == jobID {
found = op
break
}
}
Expect(found).ToNot(BeNil())
Expect(found["phase"]).To(Equal("downloading"))
Expect(found["currentBytes"]).To(Equal(float64(1024)))
Expect(found["totalBytes"]).To(Equal(float64(4096)))
})
})

View File

@@ -25,6 +25,9 @@ type GalleryOperationRecord struct {
OpType string `gorm:"size:32" json:"op_type"` // "model_install", "model_delete", "backend_install"
Status string `gorm:"size:32;default:pending" json:"status"` // pending, downloading, processing, completed, failed, cancelled
Progress float64 `json:"progress"` // 0.0 to 1.0
Phase string `gorm:"size:32" json:"phase,omitempty"`
CurrentBytes int64 `json:"current_bytes,omitempty"`
TotalBytes int64 `json:"total_bytes,omitempty"`
Message string `gorm:"type:text" json:"message,omitempty"`
Error string `gorm:"type:text" json:"error,omitempty"`
FileName string `gorm:"size:512" json:"file_name,omitempty"`
@@ -84,14 +87,26 @@ func (s *GalleryStore) Create(op *GalleryOperationRecord) error {
// op as still cancellable — otherwise the column keeps its Create-time zero
// value (false), the UI hides the cancel button, and the orphaned op can only
// be dismissed by waiting for the 30-minute stale reaper.
func (s *GalleryStore) UpdateProgress(id string, progress float64, message, downloadedSize string, cancellable bool) error {
return s.db.Model(&GalleryOperationRecord{}).Where("id = ?", id).Updates(map[string]any{
type OperationProgressDetails struct {
Phase string
CurrentBytes int64
TotalBytes int64
}
func (s *GalleryStore) UpdateProgress(id string, progress float64, message, downloadedSize string, cancellable bool, details ...OperationProgressDetails) error {
updates := map[string]any{
"progress": progress,
"message": message,
"downloaded_file_size": downloadedSize,
"cancellable": cancellable,
"updated_at": time.Now(),
}).Error
}
if len(details) > 0 {
updates["phase"] = details[0].Phase
updates["current_bytes"] = details[0].CurrentBytes
updates["total_bytes"] = details[0].TotalBytes
}
return s.db.Model(&GalleryOperationRecord{}).Where("id = ?", id).Updates(updates).Error
}
// UpdateStatus updates the status of an operation. A terminal status is never

View File

@@ -0,0 +1,65 @@
package galleryop
import (
"fmt"
"sync"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
type artifactProgressBridge struct {
mu sync.Mutex
last float64
currentBytes int64
totalBytes int64
update func(*OpStatus)
}
func newArtifactProgressBridge(update func(*OpStatus)) *artifactProgressBridge {
return &artifactProgressBridge{update: update}
}
func (b *artifactProgressBridge) Sink(event modelartifacts.ProgressEvent) {
b.mu.Lock()
progress := b.last
message := "Preparing model files"
switch event.Phase {
case modelartifacts.PhaseResolving:
progress = max(progress, 0)
message = "Resolving model files"
case modelartifacts.PhaseDownloading:
if event.TotalBytes > 0 {
progress = max(progress, min(90, float64(event.CurrentBytes)*90/float64(event.TotalBytes)))
}
message = fmt.Sprintf("Downloading model file: %s", event.File)
case modelartifacts.PhaseVerifying:
progress = max(progress, 95)
message = "Verifying model files"
case modelartifacts.PhaseCommitting:
progress = max(progress, 99)
message = "Finalizing model installation"
case modelartifacts.PhasePersisting:
progress = max(progress, 99)
message = "Saving model configuration"
}
b.last = progress
b.currentBytes = max(b.currentBytes, event.CurrentBytes)
b.totalBytes = max(b.totalBytes, event.TotalBytes)
status := &OpStatus{
Phase: string(event.Phase), Message: message, FileName: event.File,
Progress: progress, CurrentBytes: b.currentBytes, TotalBytes: b.totalBytes,
Cancellable: true,
}
update := b.update
b.mu.Unlock()
if update != nil {
update(status)
}
}
func (b *artifactProgressBridge) ClampLegacy(progress float64) float64 {
b.mu.Lock()
defer b.mu.Unlock()
b.last = max(b.last, progress)
return b.last
}

View File

@@ -0,0 +1,33 @@
package galleryop
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
var _ = Describe("artifact operation progress", func() {
It("maps phases and never moves percentage backward", func() {
var statuses []*OpStatus
bridge := newArtifactProgressBridge(func(status *OpStatus) {
copy := *status
statuses = append(statuses, &copy)
})
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseResolving})
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseDownloading, File: "model.bin", CurrentBytes: 50, TotalBytes: 100})
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseDownloading, File: "model.bin", CurrentBytes: 10, TotalBytes: 100})
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseVerifying, CurrentBytes: 100, TotalBytes: 100})
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseCommitting, CurrentBytes: 100, TotalBytes: 100})
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhasePersisting})
Expect(statuses).To(HaveLen(6))
for index := 1; index < len(statuses); index++ {
Expect(statuses[index].Progress).To(BeNumerically(">=", statuses[index-1].Progress))
}
Expect(statuses[1].Phase).To(Equal("downloading"))
Expect(statuses[1].CurrentBytes).To(Equal(int64(50)))
Expect(statuses[5].Progress).To(Equal(float64(99)))
Expect(statuses[5].CurrentBytes).To(Equal(int64(100)))
})
})

View File

@@ -38,9 +38,12 @@ var _ = Describe("GalleryService cancellable persistence across restart", func()
// Simulate a progress tick: the live path always marks installs
// cancellable while they are downloading/processing.
svc.UpdateStatus("op-inflight", &galleryop.OpStatus{
Message: "downloading",
Progress: 25,
Cancellable: true,
Message: "downloading",
Progress: 25,
Phase: "downloading",
CurrentBytes: 123,
TotalBytes: 456,
Cancellable: true,
})
// A fresh replica boots and hydrates from the store.
@@ -52,5 +55,8 @@ var _ = Describe("GalleryService cancellable persistence across restart", func()
Expect(st).ToNot(BeNil(), "the in-flight op must hydrate after a restart")
Expect(st.Cancellable).To(BeTrue(),
"a still-active install must rehydrate as cancellable so the admin can dismiss it")
Expect(st.Phase).To(Equal("downloading"))
Expect(st.CurrentBytes).To(Equal(int64(123)))
Expect(st.TotalBytes).To(Equal(int64(456)))
})
})

View File

@@ -122,6 +122,9 @@ var _ = Describe("OpStatus JSON wire format", func() {
original := &galleryop.OpStatus{
Progress: 42.0,
Message: "downloading",
Phase: "downloading",
CurrentBytes: 123,
TotalBytes: 456,
GalleryElementName: "vllm",
Error: errors.New("disk full"),
Processed: true,
@@ -134,6 +137,9 @@ var _ = Describe("OpStatus JSON wire format", func() {
Expect(got.Error).ToNot(BeNil(), "the error must survive the round-trip — peer replicas need to surface the failure")
Expect(got.Error.Error()).To(Equal("disk full"))
Expect(got.Progress).To(Equal(42.0))
Expect(got.Phase).To(Equal("downloading"))
Expect(got.CurrentBytes).To(Equal(int64(123)))
Expect(got.TotalBytes).To(Equal(int64(456)))
Expect(got.GalleryElementName).To(Equal("vllm"))
Expect(got.Processed).To(BeTrue())
})

View File

@@ -17,6 +17,7 @@ type LocalModelManager struct {
enforcePredownloadScans bool
automaticallyInstallBackend bool
requireBackendIntegrity bool
artifactMaterializer config.ArtifactMaterializer
}
// NewLocalModelManager creates a LocalModelManager from the application config.
@@ -27,6 +28,7 @@ func NewLocalModelManager(appConfig *config.ApplicationConfig, ml *model.ModelLo
enforcePredownloadScans: appConfig.EnforcePredownloadScans,
automaticallyInstallBackend: appConfig.AutoloadBackendGalleries,
requireBackendIntegrity: appConfig.RequireBackendIntegrity,
artifactMaterializer: appConfig.ModelArtifactMaterializer,
}
}
@@ -48,7 +50,8 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g
switch {
case op.GalleryElement != nil:
installedModel, err := gallery.InstallModel(ctx, m.systemState, op.GalleryElement.Name,
op.GalleryElement, op.Req.Overrides, progressCb, m.enforcePredownloadScans)
op.GalleryElement, op.Req.Overrides, progressCb, m.enforcePredownloadScans,
gallery.WithArtifactMaterializer(m.artifactMaterializer))
if err != nil {
return err
}
@@ -61,10 +64,12 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g
case op.GalleryElementName != "":
return gallery.InstallModelFromGallery(ctx, op.Galleries, op.BackendGalleries,
m.systemState, m.modelLoader, op.GalleryElementName, op.Req, progressCb,
m.enforcePredownloadScans, m.automaticallyInstallBackend, m.requireBackendIntegrity)
m.enforcePredownloadScans, m.automaticallyInstallBackend, m.requireBackendIntegrity,
gallery.WithArtifactMaterializer(m.artifactMaterializer))
default:
return installModelFromRemoteConfig(ctx, m.systemState, m.modelLoader, op.Req,
progressCb, m.enforcePredownloadScans, m.automaticallyInstallBackend, op.BackendGalleries, m.requireBackendIntegrity)
progressCb, m.enforcePredownloadScans, m.automaticallyInstallBackend, op.BackendGalleries, m.requireBackendIntegrity,
gallery.WithArtifactMaterializer(m.artifactMaterializer))
}
}

View File

@@ -0,0 +1,51 @@
package galleryop_test
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/services/galleryop"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
)
type galleryOpArtifactMaterializer struct {
seen []modelartifacts.Spec
}
func (f *galleryOpArtifactMaterializer) Ensure(_ context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
f.seen = append(f.seen, spec)
spec.Resolved = &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
}
return modelartifacts.Result{Spec: spec}, nil
}
var _ = Describe("local model manager artifact materializer", func() {
It("passes the controller materializer to direct gallery installs", func() {
state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
Expect(err).NotTo(HaveOccurred())
materializer := &galleryOpArtifactMaterializer{}
manager := galleryop.NewLocalModelManager(&config.ApplicationConfig{
SystemState: state,
ModelArtifactMaterializer: materializer,
}, nil)
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
backend: transformers
artifacts:
- source: {type: huggingface, repo: owner/repo}
parameters: {model: owner/repo}
`}
op := &galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{
GalleryElement: definition,
}
Expect(manager.InstallModel(context.Background(), op, nil)).To(Succeed())
Expect(materializer.seen).To(HaveLen(1))
})
})

View File

@@ -11,6 +11,7 @@ import (
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/services/messaging"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/modelartifacts"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
"gopkg.in/yaml.v3"
@@ -52,6 +53,16 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
g.UpdateStatus(op.ID, &OpStatus{Message: fmt.Sprintf("processing model: %s", op.GalleryElementName), Progress: 0, Cancellable: true})
bridge := newArtifactProgressBridge(func(status *OpStatus) {
status.GalleryElementName = op.GalleryElementName
g.UpdateStatus(op.ID, status)
})
operationCtx := op.Context
if operationCtx == nil {
operationCtx = context.Background()
}
operationCtx = modelartifacts.WithProgressSink(operationCtx, bridge.Sink)
// displayDownload displays the download progress
progressCallback := func(fileName string, current string, total string, percentage float64) {
// Check for cancellation during progress updates
@@ -62,6 +73,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
default:
}
}
percentage = bridge.ClampLegacy(percentage)
g.UpdateStatus(op.ID, &OpStatus{Message: fmt.Sprintf(processingMessage, fileName, total, current), FileName: fileName, Progress: percentage, TotalFileSize: total, DownloadedFileSize: current, Cancellable: true})
utils.DisplayDownloadFunction(fileName, current, total, percentage)
}
@@ -70,7 +82,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
if op.Delete {
err = g.modelManager.DeleteModel(op.GalleryElementName)
} else {
err = g.modelManager.InstallModel(op.Context, op, progressCallback)
err = g.modelManager.InstallModel(operationCtx, op, progressCallback)
}
if err != nil {
// Check if error is due to cancellation
@@ -107,7 +119,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
return err
}
err = cl.Preload(systemState.Model.ModelsPath)
err = cl.PreloadWithContext(operationCtx, systemState.Model.ModelsPath)
if err != nil {
return err
}
@@ -137,7 +149,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
return nil
}
func installModelFromRemoteConfig(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery, requireBackendIntegrity bool) error {
func installModelFromRemoteConfig(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
config, err := gallery.GetGalleryConfigFromURLWithContext[gallery.ModelConfig](ctx, req.URL, systemState.Model.ModelsPath)
if err != nil {
return err
@@ -145,7 +157,7 @@ func installModelFromRemoteConfig(ctx context.Context, systemState *system.Syste
config.Files = append(config.Files, req.AdditionalFiles...)
installedModel, err := gallery.InstallModel(ctx, systemState, req.Name, &config, req.Overrides, downloadStatus, enforceScan)
installedModel, err := gallery.InstallModel(ctx, systemState, req.Name, &config, req.Overrides, downloadStatus, enforceScan, options...)
if err != nil {
return err
}
@@ -164,23 +176,23 @@ type galleryModel struct {
ID string `json:"id"`
}
func processRequests(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel, requireBackendIntegrity bool) error {
func processRequests(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
ctx := context.Background()
var err error
for _, r := range requests {
utils.ResetDownloadTimers()
if r.ID == "" {
err = installModelFromRemoteConfig(ctx, systemState, modelLoader, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries, requireBackendIntegrity)
err = installModelFromRemoteConfig(ctx, systemState, modelLoader, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries, requireBackendIntegrity, options...)
} else {
err = gallery.InstallModelFromGallery(
ctx, galleries, backendGalleries, systemState, modelLoader, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, requireBackendIntegrity)
ctx, galleries, backendGalleries, systemState, modelLoader, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, requireBackendIntegrity, options...)
}
}
return err
}
func ApplyGalleryFromFile(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool) error {
func ApplyGalleryFromFile(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
dat, err := os.ReadFile(s)
if err != nil {
return err
@@ -191,15 +203,15 @@ func ApplyGalleryFromFile(systemState *system.SystemState, modelLoader *model.Mo
return err
}
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity)
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity, options...)
}
func ApplyGalleryFromString(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool) error {
func ApplyGalleryFromString(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
var requests []galleryModel
err := json.Unmarshal([]byte(s), &requests)
if err != nil {
return err
}
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity)
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity, options...)
}

View File

@@ -61,6 +61,9 @@ type OpStatus struct {
Processed bool `json:"processed"`
Message string `json:"message"`
Progress float64 `json:"progress"`
Phase string `json:"phase,omitempty"`
CurrentBytes int64 `json:"current_bytes,omitempty"`
TotalBytes int64 `json:"total_bytes,omitempty"`
TotalFileSize string `json:"file_size"`
DownloadedFileSize string `json:"downloaded_size"`
GalleryElementName string `json:"gallery_element_name"`
@@ -89,6 +92,9 @@ type opStatusWire struct {
Processed bool `json:"processed"`
Message string `json:"message"`
Progress float64 `json:"progress"`
Phase string `json:"phase,omitempty"`
CurrentBytes int64 `json:"current_bytes,omitempty"`
TotalBytes int64 `json:"total_bytes,omitempty"`
TotalFileSize string `json:"file_size"`
DownloadedFileSize string `json:"downloaded_size"`
GalleryElementName string `json:"gallery_element_name"`
@@ -104,6 +110,9 @@ func (o OpStatus) MarshalJSON() ([]byte, error) {
Processed: o.Processed,
Message: o.Message,
Progress: o.Progress,
Phase: o.Phase,
CurrentBytes: o.CurrentBytes,
TotalBytes: o.TotalBytes,
TotalFileSize: o.TotalFileSize,
DownloadedFileSize: o.DownloadedFileSize,
GalleryElementName: o.GalleryElementName,
@@ -127,6 +136,9 @@ func (o *OpStatus) UnmarshalJSON(data []byte) error {
o.Processed = w.Processed
o.Message = w.Message
o.Progress = w.Progress
o.Phase = w.Phase
o.CurrentBytes = w.CurrentBytes
o.TotalBytes = w.TotalBytes
o.TotalFileSize = w.TotalFileSize
o.DownloadedFileSize = w.DownloadedFileSize
o.GalleryElementName = w.GalleryElementName

View File

@@ -94,6 +94,15 @@ func (g *GalleryService) BackendManager() BackendManager {
return g.backendManager
}
// ModelArtifactMaterializer returns the controller-only acquisition capability
// used by startup paths that install gallery entries outside the operation loop.
func (g *GalleryService) ModelArtifactMaterializer() config.ArtifactMaterializer {
if g == nil || g.appConfig == nil {
return nil
}
return g.appConfig.ModelArtifactMaterializer
}
// SetNATSClient sets the NATS client for distributed progress publishing.
// Accepting the wider MessagingClient (vs. plain Publisher) lets
// SubscribeBroadcasts wire the wildcard subscriptions that keep peer
@@ -167,7 +176,10 @@ func (g *GalleryService) UpdateStatus(s string, op *OpStatus) {
xlog.Warn("Failed to persist gallery operation status", "op_id", s, "error", err)
}
} else {
if err := store.UpdateProgress(s, op.Progress, op.Message, op.DownloadedFileSize, op.Cancellable); err != nil {
if err := store.UpdateProgress(s, op.Progress, op.Message, op.DownloadedFileSize, op.Cancellable,
distributed.OperationProgressDetails{
Phase: op.Phase, CurrentBytes: op.CurrentBytes, TotalBytes: op.TotalBytes,
}); err != nil {
xlog.Warn("Failed to persist gallery operation progress", "op_id", s, "error", err)
}
}
@@ -662,6 +674,9 @@ func (g *GalleryService) Hydrate() error {
st := &OpStatus{
Message: op.Message,
Progress: op.Progress,
Phase: op.Phase,
CurrentBytes: op.CurrentBytes,
TotalBytes: op.TotalBytes,
FileName: op.FileName,
TotalFileSize: op.TotalFileSize,
DownloadedFileSize: op.DownloadedFileSize,

View File

@@ -77,4 +77,30 @@ var _ = Describe("ApplyRemoteChange", func() {
Expect(ok1).To(BeTrue())
Expect(ok2).To(BeTrue())
})
It("loads a peer-persisted artifact binding without materializing", func() {
const relative = ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot"
writeYAML("peer-managed", map[string]any{
"backend": "transformers",
"artifacts": []map[string]any{{
"name": "model", "target": "model",
"source": map[string]any{"type": "huggingface", "repo": "owner/repo", "revision": "main"},
"resolved": map[string]any{
"endpoint": "https://huggingface.co",
"revision": "0123456789abcdef0123456789abcdef01234567",
"cache_key": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
},
}},
"parameters": map[string]any{"model": "owner/repo"},
})
Expect(ApplyRemoteChange(loader, nil, dir, messaging.CacheInvalidateEvent{
Element: "peer-managed", Op: "install",
})).To(Succeed())
loaded, found := loader.GetModelConfig("peer-managed")
Expect(found).To(BeTrue())
Expect(loaded.Model).To(Equal("owner/repo"))
Expect(loaded.ModelFileName()).To(Equal(relative))
Expect(loaded.Artifacts).To(HaveLen(1))
Expect(loaded.Artifacts[0].Resolved.CacheKey).To(HaveLen(64))
})
})

View File

@@ -4,10 +4,12 @@ import (
"context"
"os"
"path/filepath"
"strings"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/services/storage"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
)
@@ -61,4 +63,48 @@ var _ = Describe("stageModelFiles directory models", func() {
Expect(staged).To(ConsistOf(weights, tokenizer))
Expect(staged).ToNot(ContainElement(modelDir))
})
It("stages a content-addressed Hugging Face snapshot with its relative tree intact", func() {
cacheKey := strings.Repeat("a", 64)
logicalModel := "owner/repo"
relativeSnapshot := filepath.Join(".artifacts", "huggingface", cacheKey, "snapshot")
snapshot := filepath.Join(tmp, "models", relativeSnapshot)
files := map[string]string{
"config.json": "{}",
"model.safetensors.index.json": "{}",
"model-00001-of-00002.safetensors": "part-1",
"model-00002-of-00002.safetensors": "part-2",
filepath.Join("tokenizer", "vocab.json"): "{}",
}
for relative, contents := range files {
path := filepath.Join(snapshot, relative)
Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
Expect(os.WriteFile(path, []byte(contents), 0o644)).To(Succeed())
}
opts := &pb.ModelOptions{Model: logicalModel, ModelFile: snapshot}
stagedOpts, err := router.stageModelFiles(context.Background(), node, opts, "managed-model")
Expect(err).NotTo(HaveOccurred())
stagedPaths := make([]string, 0, len(stager.ensureCalls))
stagedKeys := make([]string, 0, len(stager.ensureCalls))
for _, call := range stager.ensureCalls {
stagedPaths = append(stagedPaths, call.localPath)
stagedKeys = append(stagedKeys, call.key)
}
expectedPaths := make([]string, 0, len(files))
expectedKeys := make([]string, 0, len(files))
for relative := range files {
expectedPaths = append(expectedPaths, filepath.Join(snapshot, relative))
expectedKeys = append(expectedKeys, storage.ModelKey(filepath.Join("managed-model", relative)))
}
Expect(stagedPaths).To(ConsistOf(expectedPaths))
Expect(stagedKeys).To(ConsistOf(expectedKeys))
remoteSnapshot := filepath.Join("/remote", storage.ModelKey("managed-model"))
Expect(stagedOpts.Model).To(Equal(logicalModel))
Expect(stagedOpts.ModelFile).To(Equal(remoteSnapshot))
Expect(stagedOpts.ModelPath).To(Equal(remoteSnapshot))
Expect(opts.Model).To(Equal(logicalModel))
Expect(opts.ModelFile).To(Equal(snapshot))
})
})

View File

@@ -24,9 +24,13 @@ import (
func InstallModels(ctx context.Context, galleryService *galleryop.GalleryService, galleries, backendGalleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool, downloadStatus func(string, string, string, float64), models ...string) error {
// create an error that groups all errors
var err error
var installOptions []gallery.InstallOption
if galleryService != nil {
installOptions = append(installOptions, gallery.WithArtifactMaterializer(galleryService.ModelArtifactMaterializer()))
}
for _, url := range models {
// Check if it's a model gallery, or print a warning
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity)
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity, installOptions...)
if e != nil && found {
xlog.Error("[startup] failed installing model", "error", err, "model", url)
err = errors.Join(err, e)
@@ -82,7 +86,7 @@ func InstallModels(ctx context.Context, galleryService *galleryop.GalleryService
return err
}
func installModel(ctx context.Context, galleries, backendGalleries []config.Gallery, modelName string, systemState *system.SystemState, modelLoader *model.ModelLoader, downloadStatus func(string, string, string, float64), enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool) (error, bool) {
func installModel(ctx context.Context, galleries, backendGalleries []config.Gallery, modelName string, systemState *system.SystemState, modelLoader *model.ModelLoader, downloadStatus func(string, string, string, float64), enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool, options ...gallery.InstallOption) (error, bool) {
models, err := gallery.AvailableGalleryModels(galleries, systemState)
if err != nil {
return err, false
@@ -98,7 +102,7 @@ func installModel(ctx context.Context, galleries, backendGalleries []config.Gall
}
xlog.Info("installing model", "model", modelName, "license", model.License)
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity)
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity, options...)
if err != nil {
return err, true
}