mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-31 10:28:43 -04:00
feat: materialize Hugging Face model artifacts (#10825)
* feat(config): add model artifact source contract Assisted-by: Codex:GPT-5 [Codex] * feat(downloader): add authenticated raw-byte progress Assisted-by: Codex:GPT-5 [Codex] * feat(huggingface): resolve immutable snapshot manifests Assisted-by: Codex:GPT-5 [Codex] * feat(models): add artifact storage primitives Assisted-by: Codex:GPT-5 [Codex] * feat(models): materialize pinned Hugging Face snapshots Assisted-by: Codex:GPT-5 [Codex] * feat(models): bind managed snapshots at runtime Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): materialize model artifacts during install Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): declare managed Hugging Face artifacts Assisted-by: Codex:GPT-5 [Codex] * feat(models): preload managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * fix(gallery): retain shared artifact caches on delete Assisted-by: Codex:GPT-5 [Codex] * feat(models): report artifact acquisition progress Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load managed models from ModelFile Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load staged speech model snapshots Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): use staged snapshots in engine backends Assisted-by: Codex:GPT-5 [Codex] * test(distributed): cover staged artifact snapshots Assisted-by: Codex:GPT-5 [Codex] * docs: explain managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * docs: add product design context Assisted-by: Codex:GPT-5 [Codex] * feat(ui): show model artifact download progress Assisted-by: Codex:GPT-5 [Codex] * Eagerly materialize Hugging Face artifacts Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback. Assisted-by: Codex:GPT-5 [shell] * Refactor HF downloads through a shared executor Assisted-by: Codex:GPT-5 [shell] * drop Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
@@ -116,7 +116,11 @@ func newApplication(appConfig *config.ApplicationConfig) *Application {
|
||||
ml.SetLoadObserver(corebackend.ModelLoadTraceObserver(appConfig))
|
||||
|
||||
app := &Application{
|
||||
backendLoader: config.NewModelConfigLoader(appConfig.SystemState.Model.ModelsPath),
|
||||
backendLoader: config.NewModelConfigLoader(
|
||||
appConfig.SystemState.Model.ModelsPath,
|
||||
config.WithArtifactMaterializer(appConfig.ModelArtifactMaterializer),
|
||||
config.WithPreloadDisplay(appConfig.ModelPreloadRenderMode, appConfig.DisableModelPreloadColor),
|
||||
),
|
||||
modelLoader: ml,
|
||||
applicationConfig: appConfig,
|
||||
templatesEvaluator: templates.NewEvaluator(appConfig.SystemState.Model.ModelsPath),
|
||||
|
||||
55
core/application/model_artifact_materializer_test.go
Normal file
55
core/application/model_artifact_materializer_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
)
|
||||
|
||||
type applicationLoaderMaterializer struct {
|
||||
seen []modelartifacts.Spec
|
||||
}
|
||||
|
||||
func (f *applicationLoaderMaterializer) Ensure(_ context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
|
||||
f.seen = append(f.seen, spec)
|
||||
spec.Resolved = &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
}
|
||||
return modelartifacts.Result{Spec: spec}, nil
|
||||
}
|
||||
|
||||
var _ = Describe("application model artifact wiring", func() {
|
||||
It("injects the controller materializer into the model config loader", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(
|
||||
system.WithModelPath(modelsPath),
|
||||
system.WithBackendPath(GinkgoT().TempDir()),
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
materializer := &applicationLoaderMaterializer{}
|
||||
app := newApplication(&config.ApplicationConfig{
|
||||
Context: context.Background(),
|
||||
SystemState: state,
|
||||
ModelArtifactMaterializer: materializer,
|
||||
})
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, "managed.yaml"), []byte(`
|
||||
name: managed
|
||||
backend: transformers
|
||||
artifacts:
|
||||
- source: {type: huggingface, repo: owner/repo}
|
||||
parameters: {model: owner/repo}
|
||||
`), 0644)).To(Succeed())
|
||||
Expect(app.ModelConfigLoader().LoadModelConfigsFromPath(modelsPath)).To(Succeed())
|
||||
Expect(app.ModelConfigLoader().PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
|
||||
Expect(materializer.seen).To(HaveLen(1))
|
||||
})
|
||||
})
|
||||
@@ -414,18 +414,18 @@ func New(opts ...config.AppOption) (*Application, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if err := application.ModelConfigLoader().Preload(options.SystemState.Model.ModelsPath); err != nil {
|
||||
if err := application.ModelConfigLoader().PreloadWithContext(options.Context, options.SystemState.Model.ModelsPath); err != nil {
|
||||
xlog.Error("error downloading models", "error", err)
|
||||
}
|
||||
|
||||
if options.PreloadJSONModels != "" {
|
||||
if err := galleryop.ApplyGalleryFromString(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels, options.RequireBackendIntegrity); err != nil {
|
||||
if err := galleryop.ApplyGalleryFromString(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels, options.RequireBackendIntegrity, gallery.WithArtifactMaterializer(options.ModelArtifactMaterializer)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if options.PreloadModelsFromPath != "" {
|
||||
if err := galleryop.ApplyGalleryFromFile(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath, options.RequireBackendIntegrity); err != nil {
|
||||
if err := galleryop.ApplyGalleryFromFile(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath, options.RequireBackendIntegrity, gallery.WithArtifactMaterializer(options.ModelArtifactMaterializer)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,7 +109,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
|
||||
if !slices.Contains(modelNames, modelName) {
|
||||
utils.ResetDownloadTimers()
|
||||
// if we failed to load the model, we try to download it
|
||||
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, modelName, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries, o.RequireBackendIntegrity)
|
||||
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, modelName, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries, o.RequireBackendIntegrity, gallery.WithArtifactMaterializer(o.ModelArtifactMaterializer))
|
||||
if err != nil {
|
||||
xlog.Error("failed to install model from gallery", "error", err, "model", modelFile)
|
||||
//return nil, err
|
||||
|
||||
47
core/backend/model_artifact_options_test.go
Normal file
47
core/backend/model_artifact_options_test.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
var _ = Describe("managed model runtime options", func() {
|
||||
It("estimates weights from the committed manifest without repository fallback", func() {
|
||||
const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
spec := modelartifacts.Spec{
|
||||
Name: "model", Target: "model",
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: cacheKey,
|
||||
},
|
||||
}
|
||||
relative, err := modelartifacts.RelativeSnapshotPath(cacheKey)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
manifest := modelartifacts.Manifest{
|
||||
Version: modelartifacts.ManifestVersion,
|
||||
Artifact: spec,
|
||||
Files: []modelartifacts.ManifestFile{{
|
||||
Path: "weights/model.safetensors", Size: 12, SHA256: strings.Repeat("a", 64),
|
||||
}},
|
||||
}
|
||||
encoded, err := json.Marshal(manifest)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
manifestPath := filepath.Join(modelsPath, filepath.Dir(relative), "manifest.json")
|
||||
Expect(os.MkdirAll(filepath.Dir(manifestPath), 0750)).To(Succeed())
|
||||
Expect(os.WriteFile(manifestPath, encoded, 0644)).To(Succeed())
|
||||
|
||||
cfg := config.ModelConfig{Artifacts: []modelartifacts.Spec{spec}}
|
||||
Expect(estimateModelSizeBytes(cfg, modelsPath)).To(Equal(int64(12)))
|
||||
})
|
||||
})
|
||||
@@ -12,9 +12,10 @@ import (
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/trace"
|
||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/vram"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
@@ -93,8 +94,9 @@ func recordModelLoadFailure(appConfig *config.ApplicationConfig, modelName, back
|
||||
func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
|
||||
seen := make(map[string]bool)
|
||||
input := vram.ModelEstimateInput{}
|
||||
managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil
|
||||
|
||||
addFile := func(uri string) {
|
||||
addFile := func(uri string, size int64) {
|
||||
if !vram.IsWeightFile(uri) {
|
||||
return
|
||||
}
|
||||
@@ -106,7 +108,7 @@ func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
|
||||
return
|
||||
}
|
||||
seen[resolved] = true
|
||||
input.Files = append(input.Files, vram.FileInput{URI: resolved})
|
||||
input.Files = append(input.Files, vram.FileInput{URI: resolved, Size: size})
|
||||
}
|
||||
|
||||
// tryHFRepo resolves any huggingface:// or hf:// URI to an HTTPS URL and
|
||||
@@ -123,13 +125,25 @@ func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
|
||||
|
||||
for _, f := range c.DownloadFiles {
|
||||
uriStr := string(f.URI)
|
||||
addFile(uriStr)
|
||||
tryHFRepo(uriStr)
|
||||
addFile(uriStr, 0)
|
||||
if !managedPrimary {
|
||||
tryHFRepo(uriStr)
|
||||
}
|
||||
}
|
||||
if managedPrimary {
|
||||
relative := c.ModelFileName()
|
||||
manifest, err := modelartifacts.ReadManifest(filepath.Join(modelsPath, filepath.Dir(relative), "manifest.json"))
|
||||
if err == nil {
|
||||
for _, file := range manifest.Files {
|
||||
addFile(filepath.Join(relative, filepath.FromSlash(file.Path)), file.Size)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
addFile(c.Model, 0)
|
||||
tryHFRepo(c.Model)
|
||||
}
|
||||
addFile(c.Model)
|
||||
tryHFRepo(c.Model)
|
||||
if c.MMProj != "" {
|
||||
addFile(c.MMProj)
|
||||
addFile(c.MMProj, 0)
|
||||
}
|
||||
|
||||
if len(input.Files) == 0 && input.HFRepo == "" {
|
||||
@@ -153,6 +167,10 @@ func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...mo
|
||||
model.WithContext(so.Context),
|
||||
model.WithModelID(c.ModelID()),
|
||||
}
|
||||
managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil
|
||||
if managedPrimary {
|
||||
defOpts = append(defOpts, model.WithModelFile(c.ModelFileName()))
|
||||
}
|
||||
|
||||
threads := 1
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/startup"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
"github.com/mudler/xlog"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
@@ -24,6 +25,9 @@ type ModelsCMDFlags struct {
|
||||
BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"`
|
||||
ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
|
||||
BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"storage"`
|
||||
Color string `env:"COLOR" hidden:""`
|
||||
NoColor string `env:"NO_COLOR" hidden:""`
|
||||
HFToken string `env:"HF_TOKEN" hidden:""`
|
||||
}
|
||||
|
||||
type ModelsList struct {
|
||||
@@ -80,10 +84,18 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
artifactMaterializer := modelartifacts.NewDefaultManager(
|
||||
modelartifacts.WithHuggingFaceToken(mi.HFToken),
|
||||
)
|
||||
galleryService := galleryop.NewGalleryService(&config.ApplicationConfig{
|
||||
SystemState: systemState,
|
||||
SystemState: systemState,
|
||||
ModelArtifactMaterializer: artifactMaterializer,
|
||||
ModelPreloadRenderMode: mi.Color,
|
||||
DisableModelPreloadColor: mi.NoColor != "",
|
||||
}, model.NewModelLoader(systemState))
|
||||
err = galleryService.Start(context.Background(), config.NewModelConfigLoader(mi.ModelsPath), systemState)
|
||||
err = galleryService.Start(context.Background(), config.NewModelConfigLoader(mi.ModelsPath,
|
||||
config.WithArtifactMaterializer(artifactMaterializer),
|
||||
config.WithPreloadDisplay(mi.Color, mi.NoColor != "")), systemState)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/http"
|
||||
"github.com/mudler/LocalAI/core/p2p"
|
||||
"github.com/mudler/LocalAI/internal"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/signals"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
"github.com/mudler/xlog"
|
||||
@@ -28,6 +29,9 @@ import (
|
||||
|
||||
type RunCMD struct {
|
||||
ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"`
|
||||
Color string `env:"COLOR" hidden:""`
|
||||
NoColor string `env:"NO_COLOR" hidden:""`
|
||||
HFToken string `env:"HF_TOKEN" hidden:""`
|
||||
|
||||
ExternalBackends []string `env:"LOCALAI_EXTERNAL_BACKENDS,EXTERNAL_BACKENDS" help:"A list of external backends to load from gallery on boot" group:"backends"`
|
||||
WebRTCNAT1To1IPs []string `env:"LOCALAI_WEBRTC_NAT_1TO1_IPS,WEBRTC_NAT_1TO1_IPS" help:"IPs advertised as the host ICE candidates for /v1/realtime WebRTC instead of every local interface. Set to the reachable host/LAN IP when running under Docker host networking or NAT, where pion otherwise offers unreachable bridge addresses and the connection drops after ICE consent checks fail." group:"api"`
|
||||
@@ -238,6 +242,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
|
||||
|
||||
opts := []config.AppOption{
|
||||
config.WithContext(context.Background()),
|
||||
config.WithModelArtifactMaterializer(modelartifacts.NewDefaultManager(
|
||||
modelartifacts.WithHuggingFaceToken(r.HFToken),
|
||||
)),
|
||||
config.WithModelPreloadDisplay(r.Color, r.NoColor != ""),
|
||||
config.WithConfigFile(r.ModelsConfigFile),
|
||||
config.WithJSONStringPreload(r.PreloadModels),
|
||||
config.WithYAMLConfigPreload(r.PreloadModelsConfig),
|
||||
|
||||
34
core/config/application_artifact_materializer_test.go
Normal file
34
core/config/application_artifact_materializer_test.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
type applicationArtifactMaterializer struct{}
|
||||
|
||||
func (*applicationArtifactMaterializer) Ensure(context.Context, string, modelartifacts.Spec) (modelartifacts.Result, error) {
|
||||
return modelartifacts.Result{}, nil
|
||||
}
|
||||
|
||||
var _ = Describe("ApplicationConfig model artifact materializer", func() {
|
||||
It("provides a default materializer", func() {
|
||||
Expect(NewApplicationConfig().ModelArtifactMaterializer).NotTo(BeNil())
|
||||
})
|
||||
|
||||
It("accepts an injected materializer without exposing it to serialization", func() {
|
||||
materializer := &applicationArtifactMaterializer{}
|
||||
appConfig := NewApplicationConfig(WithModelArtifactMaterializer(materializer))
|
||||
Expect(appConfig.ModelArtifactMaterializer).To(BeIdenticalTo(materializer))
|
||||
|
||||
field, found := reflect.TypeOf(ApplicationConfig{}).FieldByName("ModelArtifactMaterializer")
|
||||
Expect(found).To(BeTrue())
|
||||
Expect(field.Tag.Get("json")).To(Equal("-"))
|
||||
Expect(field.Tag.Get("yaml")).To(Equal("-"))
|
||||
})
|
||||
})
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||
"github.com/mudler/xlog"
|
||||
@@ -18,6 +19,13 @@ type ApplicationConfig struct {
|
||||
SystemState *system.SystemState
|
||||
ExternalBackends []string
|
||||
|
||||
// ModelArtifactMaterializer is a controller-only acquisition capability.
|
||||
// It is excluded from serialization so credentials captured by its concrete
|
||||
// implementation cannot enter persisted settings or distributed payloads.
|
||||
ModelArtifactMaterializer ArtifactMaterializer `json:"-" yaml:"-"`
|
||||
ModelPreloadRenderMode string `json:"-" yaml:"-"`
|
||||
DisableModelPreloadColor bool `json:"-" yaml:"-"`
|
||||
|
||||
// WebRTCNAT1To1IPs, when set, are advertised as the host ICE candidates for
|
||||
// /v1/realtime WebRTC instead of every local interface address. Needed when
|
||||
// the routable address differs from what pion gathers — e.g. Docker host
|
||||
@@ -246,9 +254,11 @@ type AppOption func(*ApplicationConfig)
|
||||
|
||||
func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
|
||||
opt := &ApplicationConfig{
|
||||
Context: context.Background(),
|
||||
UploadLimitMB: 15,
|
||||
Debug: true,
|
||||
Context: context.Background(),
|
||||
UploadLimitMB: 15,
|
||||
Debug: true,
|
||||
ModelArtifactMaterializer: modelartifacts.NewDefaultManager(),
|
||||
ModelPreloadRenderMode: "dark",
|
||||
// Capture backend process stdout/stderr into the per-model
|
||||
// BackendLogStore by default so the UI "Backend Logs" page works out
|
||||
// of the box in single mode, matching worker/distributed mode (which
|
||||
@@ -256,9 +266,9 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
|
||||
// toggle can still turn it off (a persisted false wins - see
|
||||
// loadRuntimeSettingsFromFile).
|
||||
EnableBackendLogging: true,
|
||||
AgentJobRetentionDays: 30, // Default: 30 days
|
||||
LRUEvictionMaxRetries: 30, // Default: 30 retries
|
||||
LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
|
||||
AgentJobRetentionDays: 30, // Default: 30 days
|
||||
LRUEvictionMaxRetries: 30, // Default: 30 retries
|
||||
LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
|
||||
ModelLoadFailureCooldown: 10 * time.Second, // Default: 10s base cooldown after a failed load
|
||||
// WatchDogInterval is intentionally left at the zero value here.
|
||||
// The startup loader applies a persisted runtime_settings.json value
|
||||
@@ -636,6 +646,25 @@ func WithContext(ctx context.Context) AppOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithModelArtifactMaterializer injects the controller-only model acquisition capability.
|
||||
func WithModelArtifactMaterializer(materializer ArtifactMaterializer) AppOption {
|
||||
return func(o *ApplicationConfig) {
|
||||
if materializer != nil {
|
||||
o.ModelArtifactMaterializer = materializer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithModelPreloadDisplay configures terminal rendering for model preload output.
|
||||
func WithModelPreloadDisplay(renderMode string, disableColor bool) AppOption {
|
||||
return func(o *ApplicationConfig) {
|
||||
if renderMode != "" {
|
||||
o.ModelPreloadRenderMode = renderMode
|
||||
}
|
||||
o.DisableModelPreloadColor = disableColor
|
||||
}
|
||||
}
|
||||
|
||||
func WithYAMLConfigPreload(configFile string) AppOption {
|
||||
return func(o *ApplicationConfig) {
|
||||
o.PreloadModelsFromPath = configFile
|
||||
|
||||
@@ -82,6 +82,14 @@ func DefaultRegistry() map[string]FieldMetaOverride {
|
||||
Options: ModalityOptions,
|
||||
Order: 8,
|
||||
},
|
||||
"artifacts": {
|
||||
Section: "general",
|
||||
Label: "Managed Model Artifacts",
|
||||
Description: "Controller-managed model sources. LocalAI resolves, downloads, verifies, and binds these sources before a backend loads.",
|
||||
Component: "json-editor",
|
||||
Advanced: true,
|
||||
Order: 9,
|
||||
},
|
||||
|
||||
// --- LLM ---
|
||||
"context_size": {
|
||||
|
||||
110
core/config/model_artifact_binding.go
Normal file
110
core/config/model_artifact_binding.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
func persistArtifactBinding(fileName, modelName string, result modelartifacts.Result) error {
|
||||
data, err := os.ReadFile(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var document yaml.Node
|
||||
if err := yaml.Unmarshal(data, &document); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := findModelMapping(&document, modelName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
artifactValue := &yaml.Node{}
|
||||
encoded, err := yaml.Marshal([]modelartifacts.Spec{result.Spec})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := yaml.Unmarshal(encoded, artifactValue); err != nil {
|
||||
return err
|
||||
}
|
||||
setMappingValue(target, "artifacts", artifactValue.Content[0])
|
||||
updated, err := yaml.Marshal(&document)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeBindingAtomic(fileName, updated)
|
||||
}
|
||||
|
||||
func findModelMapping(document *yaml.Node, modelName string) (*yaml.Node, error) {
|
||||
if len(document.Content) != 1 {
|
||||
return nil, fmt.Errorf("invalid model configuration document")
|
||||
}
|
||||
root := document.Content[0]
|
||||
if root.Kind == yaml.MappingNode {
|
||||
return root, nil
|
||||
}
|
||||
if root.Kind == yaml.SequenceNode {
|
||||
for _, candidate := range root.Content {
|
||||
if candidate.Kind == yaml.MappingNode {
|
||||
name := mappingValue(candidate, "name")
|
||||
if name != nil && name.Value == modelName {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("model %q not found in configuration document", modelName)
|
||||
}
|
||||
|
||||
func mappingValue(mapping *yaml.Node, key string) *yaml.Node {
|
||||
for index := 0; index+1 < len(mapping.Content); index += 2 {
|
||||
if mapping.Content[index].Value == key {
|
||||
return mapping.Content[index+1]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setMappingValue(mapping *yaml.Node, key string, value *yaml.Node) {
|
||||
for index := 0; index+1 < len(mapping.Content); index += 2 {
|
||||
if mapping.Content[index].Value == key {
|
||||
mapping.Content[index+1] = value
|
||||
return
|
||||
}
|
||||
}
|
||||
mapping.Content = append(mapping.Content,
|
||||
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, value,
|
||||
)
|
||||
}
|
||||
|
||||
func writeBindingAtomic(fileName string, data []byte) error {
|
||||
temporary, err := os.CreateTemp(filepath.Dir(fileName), ".artifact-binding-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryName := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryName) }()
|
||||
if err := temporary.Chmod(0600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(data); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(temporaryName, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temporaryName, fileName)
|
||||
}
|
||||
50
core/config/model_artifact_binding_test.go
Normal file
50
core/config/model_artifact_binding_test.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
var _ = Describe("artifact binding persistence", func() {
|
||||
It("updates only the named model in a multi-config document", func() {
|
||||
fileName := filepath.Join(GinkgoT().TempDir(), "models.yaml")
|
||||
Expect(os.WriteFile(fileName, []byte(`
|
||||
- name: managed
|
||||
backend: transformers
|
||||
unknown_extension: keep-me
|
||||
artifacts:
|
||||
- source: {type: huggingface, repo: owner/repo}
|
||||
parameters: {model: owner/repo}
|
||||
- name: sibling
|
||||
backend: llama-cpp
|
||||
sibling_only: true
|
||||
parameters: {model: sibling.gguf}
|
||||
`), 0644)).To(Succeed())
|
||||
result := modelartifacts.Result{
|
||||
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
|
||||
Spec: modelartifacts.Spec{
|
||||
Name: "model", Target: "model",
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
},
|
||||
},
|
||||
}
|
||||
Expect(persistArtifactBinding(fileName, "managed", result)).To(Succeed())
|
||||
updated, err := os.ReadFile(fileName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(updated)).To(ContainSubstring("name: sibling"))
|
||||
Expect(string(updated)).To(ContainSubstring("sibling_only: true"))
|
||||
Expect(string(updated)).To(ContainSubstring("unknown_extension: keep-me"))
|
||||
Expect(string(updated)).To(ContainSubstring("model: owner/repo"))
|
||||
Expect(string(updated)).To(ContainSubstring("cache_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
|
||||
Expect(string(updated)).To(ContainSubstring("revision: 0123456789abcdef0123456789abcdef01234567"))
|
||||
})
|
||||
})
|
||||
61
core/config/model_artifact_inference.go
Normal file
61
core/config/model_artifact_inference.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
// PrimaryArtifactSpec returns the managed primary artifact to materialize for
|
||||
// this config. The boolean return is false when the config should stay on the
|
||||
// legacy path.
|
||||
func (c ModelConfig) PrimaryArtifactSpec(modelsPath string) (modelartifacts.Spec, bool, bool, error) {
|
||||
if len(c.Artifacts) > 0 {
|
||||
return c.Artifacts[0], false, true, nil
|
||||
}
|
||||
|
||||
modelRef := strings.TrimSpace(c.Model)
|
||||
if modelRef == "" {
|
||||
return modelartifacts.Spec{}, false, false, nil
|
||||
}
|
||||
if len(c.DownloadFiles) > 0 {
|
||||
return modelartifacts.Spec{}, false, false, nil
|
||||
}
|
||||
|
||||
if modelsPath != "" {
|
||||
for _, candidate := range []string{
|
||||
modelRef,
|
||||
filepath.Join(modelsPath, modelRef),
|
||||
} {
|
||||
if info, err := os.Stat(candidate); err == nil && (info.IsDir() || info.Mode().IsRegular()) {
|
||||
return modelartifacts.Spec{}, false, false, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
spec, ok, err := modelartifacts.ParsePrimaryReference(modelRef)
|
||||
if err != nil {
|
||||
return modelartifacts.Spec{}, false, false, err
|
||||
}
|
||||
if !ok {
|
||||
if strings.Contains(modelRef, "/") && !strings.HasPrefix(modelRef, ".") && !filepath.IsAbs(modelRef) {
|
||||
repo, err := modelartifacts.CanonicalRepo(modelRef)
|
||||
if err != nil {
|
||||
if strings.Count(modelRef, "/") == 1 {
|
||||
return modelartifacts.Spec{}, false, false, fmt.Errorf("invalid Hugging Face reference %q: %w", modelRef, err)
|
||||
}
|
||||
return modelartifacts.Spec{}, false, false, nil
|
||||
}
|
||||
return modelartifacts.Spec{
|
||||
Name: modelartifacts.TargetModel,
|
||||
Target: modelartifacts.TargetModel,
|
||||
Source: modelartifacts.Source{Type: modelartifacts.SourceTypeHuggingFace, Repo: repo},
|
||||
}, true, true, nil
|
||||
}
|
||||
return modelartifacts.Spec{}, false, false, nil
|
||||
}
|
||||
return spec, true, true, nil
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/services/routing/piipattern"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
"github.com/mudler/LocalAI/pkg/functions"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/reasoning"
|
||||
"github.com/mudler/cogito"
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -40,7 +41,8 @@ type ModelConfig struct {
|
||||
modelConfigFile string `yaml:"-" json:"-"`
|
||||
modelTemplate string `yaml:"-" json:"-"`
|
||||
schema.PredictionOptions `yaml:"parameters,omitempty" json:"parameters,omitempty"`
|
||||
Name string `yaml:"name,omitempty" json:"name,omitempty"`
|
||||
Name string `yaml:"name,omitempty" json:"name,omitempty"`
|
||||
Artifacts []modelartifacts.Spec `yaml:"artifacts,omitempty" json:"artifacts,omitempty"`
|
||||
|
||||
// Alias, when set, makes this config a pure redirect: every request for
|
||||
// Name is served by the model named here. All other fields are ignored.
|
||||
@@ -1210,9 +1212,15 @@ func (c ModelConfig) ModelID() string {
|
||||
return c.Model
|
||||
}
|
||||
|
||||
// ModelFileName returns the filename of the model
|
||||
// If the model is a URL, it will return the MD5 of the URL which is the filename
|
||||
// ModelFileName returns the controller-managed snapshot when the model has a
|
||||
// committed artifact, otherwise preserving the legacy URL/repository behavior.
|
||||
func (c *ModelConfig) ModelFileName() string {
|
||||
if len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil {
|
||||
relative, err := modelartifacts.RelativeSnapshotPath(c.Artifacts[0].Resolved.CacheKey)
|
||||
if err == nil {
|
||||
return relative
|
||||
}
|
||||
}
|
||||
uri := downloader.URI(c.Model)
|
||||
if uri.LooksLikeURL() {
|
||||
f, _ := uri.FilenameFromUrl()
|
||||
@@ -1315,6 +1323,21 @@ func (cfg *ModelConfig) SetDefaults(opts ...ConfigLoaderOption) {
|
||||
}
|
||||
|
||||
func (c *ModelConfig) Validate() (bool, error) {
|
||||
if c.IsAlias() && len(c.Artifacts) > 0 {
|
||||
return false, fmt.Errorf("alias model %q cannot declare artifacts", c.Name)
|
||||
}
|
||||
seenArtifacts := make(map[string]struct{}, len(c.Artifacts))
|
||||
for i, artifact := range c.Artifacts {
|
||||
normalized, err := artifact.Normalize()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("artifact %d: %w", i, err)
|
||||
}
|
||||
if _, exists := seenArtifacts[normalized.Name]; exists {
|
||||
return false, fmt.Errorf("duplicate artifact name %q", normalized.Name)
|
||||
}
|
||||
seenArtifacts[normalized.Name] = struct{}{}
|
||||
}
|
||||
|
||||
// An alias is a pure redirect: validate only its own shape here. Target
|
||||
// existence and the no-chain rule need the full config set, so the loader
|
||||
// (load-time) and the create/swap endpoints enforce those.
|
||||
|
||||
@@ -2,11 +2,13 @@ package config
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -14,22 +16,59 @@ import (
|
||||
"github.com/charmbracelet/glamour"
|
||||
"github.com/mudler/LocalAI/core/schema"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/utils"
|
||||
"github.com/mudler/xlog"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// ArtifactMaterializer resolves and commits a model artifact into local storage.
|
||||
type ArtifactMaterializer interface {
|
||||
Ensure(context.Context, string, modelartifacts.Spec) (modelartifacts.Result, error)
|
||||
}
|
||||
|
||||
// ModelConfigLoaderOption customizes a ModelConfigLoader at construction time.
|
||||
type ModelConfigLoaderOption func(*ModelConfigLoader)
|
||||
|
||||
// WithArtifactMaterializer sets the controller-side artifact acquisition boundary.
|
||||
func WithArtifactMaterializer(materializer ArtifactMaterializer) ModelConfigLoaderOption {
|
||||
return func(loader *ModelConfigLoader) {
|
||||
if materializer != nil {
|
||||
loader.artifactMaterializer = materializer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithPreloadDisplay configures terminal rendering without reading process state.
|
||||
func WithPreloadDisplay(renderMode string, disableColor bool) ModelConfigLoaderOption {
|
||||
return func(loader *ModelConfigLoader) {
|
||||
if renderMode != "" {
|
||||
loader.preloadRenderMode = renderMode
|
||||
}
|
||||
loader.disablePreloadColor = disableColor
|
||||
}
|
||||
}
|
||||
|
||||
type ModelConfigLoader struct {
|
||||
configs map[string]ModelConfig
|
||||
modelPath string
|
||||
configs map[string]ModelConfig
|
||||
modelPath string
|
||||
artifactMaterializer ArtifactMaterializer
|
||||
preloadRenderMode string
|
||||
disablePreloadColor bool
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func NewModelConfigLoader(modelPath string) *ModelConfigLoader {
|
||||
return &ModelConfigLoader{
|
||||
configs: make(map[string]ModelConfig),
|
||||
modelPath: modelPath,
|
||||
func NewModelConfigLoader(modelPath string, options ...ModelConfigLoaderOption) *ModelConfigLoader {
|
||||
loader := &ModelConfigLoader{
|
||||
configs: make(map[string]ModelConfig),
|
||||
modelPath: modelPath,
|
||||
artifactMaterializer: modelartifacts.NewDefaultManager(),
|
||||
preloadRenderMode: "dark",
|
||||
}
|
||||
for _, option := range options {
|
||||
option(loader)
|
||||
}
|
||||
return loader
|
||||
}
|
||||
|
||||
type LoadOptions struct {
|
||||
@@ -351,98 +390,170 @@ func (bcl *ModelConfigLoader) ValidateAliasTarget(cfg *ModelConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Preload prepare models if they are not local but url or huggingface repositories
|
||||
type preloadWork struct {
|
||||
key string
|
||||
config ModelConfig
|
||||
}
|
||||
|
||||
// Preload prepares models if they are not local but URLs or Hugging Face repositories.
|
||||
func (bcl *ModelConfigLoader) Preload(modelPath string) error {
|
||||
return bcl.PreloadWithContext(context.Background(), modelPath)
|
||||
}
|
||||
|
||||
// PreloadWithContext prepares remote model inputs while honoring cancellation.
|
||||
func (bcl *ModelConfigLoader) PreloadWithContext(ctx context.Context, modelPath string) error {
|
||||
bcl.Lock()
|
||||
defer bcl.Unlock()
|
||||
work := make([]preloadWork, 0, len(bcl.configs))
|
||||
for key, config := range bcl.configs {
|
||||
configCopy := config
|
||||
configCopy.Artifacts = slices.Clone(config.Artifacts)
|
||||
configCopy.DownloadFiles = slices.Clone(config.DownloadFiles)
|
||||
work = append(work, preloadWork{key: key, config: configCopy})
|
||||
}
|
||||
bcl.Unlock()
|
||||
|
||||
status := func(fileName, current, total string, percent float64) {
|
||||
utils.DisplayDownloadFunction(fileName, current, total, percent)
|
||||
}
|
||||
|
||||
xlog.Info("Preloading models", "path", modelPath)
|
||||
for _, item := range work {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
updated, artifactResult, err := bcl.preloadOne(ctx, modelPath, item.config, status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
renderMode := "dark"
|
||||
if os.Getenv("COLOR") != "" {
|
||||
renderMode = os.Getenv("COLOR")
|
||||
bcl.Lock()
|
||||
current, exists := bcl.configs[item.key]
|
||||
if !exists || !reflect.DeepEqual(current, item.config) {
|
||||
bcl.Unlock()
|
||||
continue
|
||||
}
|
||||
if artifactResult != nil && bindingNeedsPersistence(current, *artifactResult) && current.modelConfigFile != "" {
|
||||
modelartifacts.ReportProgress(ctx, modelartifacts.ProgressEvent{
|
||||
Phase: modelartifacts.PhasePersisting,
|
||||
Artifact: artifactResult.Spec.Name,
|
||||
})
|
||||
if err := persistArtifactBinding(current.modelConfigFile, current.Name, *artifactResult); err != nil {
|
||||
bcl.Unlock()
|
||||
return err
|
||||
}
|
||||
}
|
||||
bcl.configs[item.key] = updated
|
||||
bcl.Unlock()
|
||||
bcl.displayPreloadedModel(updated)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bcl *ModelConfigLoader) preloadOne(
|
||||
ctx context.Context,
|
||||
modelPath string,
|
||||
config ModelConfig,
|
||||
status func(string, string, string, float64),
|
||||
) (ModelConfig, *modelartifacts.Result, error) {
|
||||
updated := config
|
||||
updated.Artifacts = slices.Clone(config.Artifacts)
|
||||
tasks := make([]downloader.FileTask, 0, len(updated.DownloadFiles))
|
||||
for index, file := range updated.DownloadFiles {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ModelConfig{}, nil, err
|
||||
}
|
||||
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
|
||||
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
|
||||
return ModelConfig{}, nil, err
|
||||
}
|
||||
tasks = append(tasks, downloader.FileTask{
|
||||
URI: file.URI,
|
||||
Destination: filepath.Join(modelPath, file.Filename),
|
||||
SHA256: file.SHA256,
|
||||
FileIndex: index,
|
||||
TotalFiles: len(updated.DownloadFiles),
|
||||
})
|
||||
}
|
||||
if err := downloader.DownloadFilesWithContext(ctx, tasks, status); err != nil {
|
||||
return ModelConfig{}, nil, err
|
||||
}
|
||||
|
||||
artifactSpec, inferred, managedPrimary, err := updated.PrimaryArtifactSpec(modelPath)
|
||||
if err != nil {
|
||||
return ModelConfig{}, nil, err
|
||||
}
|
||||
var artifactResult *modelartifacts.Result
|
||||
if managedPrimary {
|
||||
result, err := bcl.artifactMaterializer.Ensure(ctx, modelPath, artifactSpec)
|
||||
if err != nil {
|
||||
if inferred {
|
||||
xlog.Warn("falling back to legacy model loading after artifact materialization failed", "model", updated.Name, "error", err)
|
||||
managedPrimary = false
|
||||
} else {
|
||||
return ModelConfig{}, nil, err
|
||||
}
|
||||
} else {
|
||||
next := []modelartifacts.Spec{result.Spec}
|
||||
if len(updated.Artifacts) > 1 {
|
||||
next = append(next, updated.Artifacts[1:]...)
|
||||
}
|
||||
updated.Artifacts = next
|
||||
artifactResult = &result
|
||||
}
|
||||
}
|
||||
|
||||
if !managedPrimary && updated.IsModelURL() {
|
||||
modelFileName := updated.ModelFileName()
|
||||
uri := downloader.URI(updated.Model)
|
||||
if uri.ResolveURL() != updated.Model {
|
||||
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
|
||||
if err := uri.DownloadFileWithContext(ctx, filepath.Join(modelPath, modelFileName), "", 0, 0, status); err != nil {
|
||||
return ModelConfig{}, nil, err
|
||||
}
|
||||
}
|
||||
updated.Model = modelFileName
|
||||
}
|
||||
}
|
||||
|
||||
if updated.IsMMProjURL() {
|
||||
modelFileName := updated.MMProjFileName()
|
||||
uri := downloader.URI(updated.MMProj)
|
||||
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
|
||||
if err := uri.DownloadFileWithContext(ctx, filepath.Join(modelPath, modelFileName), "", 0, 0, status); err != nil {
|
||||
return ModelConfig{}, nil, err
|
||||
}
|
||||
}
|
||||
updated.MMProj = modelFileName
|
||||
}
|
||||
return updated, artifactResult, nil
|
||||
}
|
||||
|
||||
func bindingNeedsPersistence(current ModelConfig, result modelartifacts.Result) bool {
|
||||
return len(current.Artifacts) == 0 || !reflect.DeepEqual(current.Artifacts[0], result.Spec)
|
||||
}
|
||||
|
||||
func (bcl *ModelConfigLoader) displayPreloadedModel(config ModelConfig) {
|
||||
glamText := func(t string) {
|
||||
out, err := glamour.Render(t, renderMode)
|
||||
if err == nil && os.Getenv("NO_COLOR") == "" {
|
||||
out, err := glamour.Render(t, bcl.preloadRenderMode)
|
||||
if err == nil && !bcl.disablePreloadColor {
|
||||
fmt.Println(out)
|
||||
} else {
|
||||
fmt.Println(t)
|
||||
}
|
||||
}
|
||||
|
||||
for i, config := range bcl.configs {
|
||||
|
||||
// Download files and verify their SHA
|
||||
for i, file := range config.DownloadFiles {
|
||||
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
|
||||
|
||||
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
|
||||
return err
|
||||
}
|
||||
// Create file path
|
||||
filePath := filepath.Join(modelPath, file.Filename)
|
||||
|
||||
if err := file.URI.DownloadFile(filePath, file.SHA256, i, len(config.DownloadFiles), status); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// If the model is an URL, expand it, and download the file
|
||||
if config.IsModelURL() {
|
||||
modelFileName := config.ModelFileName()
|
||||
uri := downloader.URI(config.Model)
|
||||
if uri.ResolveURL() != config.Model {
|
||||
// check if file exists
|
||||
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
|
||||
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cc := bcl.configs[i]
|
||||
c := &cc
|
||||
c.PredictionOptions.Model = modelFileName
|
||||
bcl.configs[i] = *c
|
||||
}
|
||||
}
|
||||
|
||||
if config.IsMMProjURL() {
|
||||
modelFileName := config.MMProjFileName()
|
||||
uri := downloader.URI(config.MMProj)
|
||||
// check if file exists
|
||||
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
|
||||
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
cc := bcl.configs[i]
|
||||
c := &cc
|
||||
c.MMProj = modelFileName
|
||||
bcl.configs[i] = *c
|
||||
}
|
||||
|
||||
if bcl.configs[i].Name != "" {
|
||||
glamText(fmt.Sprintf("**Model name**: _%s_", bcl.configs[i].Name))
|
||||
}
|
||||
if bcl.configs[i].Description != "" {
|
||||
//glamText("**Description**")
|
||||
glamText(bcl.configs[i].Description)
|
||||
}
|
||||
if bcl.configs[i].Usage != "" {
|
||||
//glamText("**Usage**")
|
||||
glamText(bcl.configs[i].Usage)
|
||||
}
|
||||
if config.Name != "" {
|
||||
glamText(fmt.Sprintf("**Model name**: _%s_", config.Name))
|
||||
}
|
||||
if config.Description != "" {
|
||||
glamText(config.Description)
|
||||
}
|
||||
if config.Usage != "" {
|
||||
glamText(config.Usage)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MITMHostOwnership is the result of mapping intercept hosts to the
|
||||
|
||||
@@ -1,10 +1,215 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
type preloadArtifactMaterializer struct {
|
||||
result modelartifacts.Result
|
||||
err error
|
||||
seen chan modelartifacts.Spec
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (f *preloadArtifactMaterializer) Ensure(ctx context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
|
||||
if f.seen != nil {
|
||||
f.seen <- spec
|
||||
}
|
||||
if f.release != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return modelartifacts.Result{}, ctx.Err()
|
||||
case <-f.release:
|
||||
}
|
||||
}
|
||||
return f.result, f.err
|
||||
}
|
||||
|
||||
var _ = Describe("ModelConfigLoader artifact preload", func() {
|
||||
It("materializes and persists a source-only artifact binding", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
configPath := filepath.Join(modelsPath, "managed.yaml")
|
||||
Expect(os.WriteFile(configPath, []byte(`
|
||||
name: managed
|
||||
backend: transformers
|
||||
unknown_extension: keep-me
|
||||
artifacts:
|
||||
- name: model
|
||||
target: model
|
||||
source: {type: huggingface, repo: owner/repo}
|
||||
parameters: {model: owner/repo}
|
||||
`), 0644)).To(Succeed())
|
||||
resolved := modelartifacts.Spec{
|
||||
Name: "model", Target: "model",
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
},
|
||||
}
|
||||
fake := &preloadArtifactMaterializer{result: modelartifacts.Result{
|
||||
Spec: resolved,
|
||||
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
|
||||
}, seen: make(chan modelartifacts.Spec, 1)}
|
||||
loader := NewModelConfigLoader(modelsPath, WithArtifactMaterializer(fake))
|
||||
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
|
||||
Expect(loader.PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
|
||||
|
||||
loaded, found := loader.GetModelConfig("managed")
|
||||
Expect(found).To(BeTrue())
|
||||
Expect(loaded.Model).To(Equal("owner/repo"))
|
||||
Expect(loaded.ModelFileName()).To(Equal(fake.result.RelativePath))
|
||||
data, err := os.ReadFile(configPath)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(data)).To(ContainSubstring("unknown_extension: keep-me"))
|
||||
Expect(string(data)).To(ContainSubstring("revision: 0123456789abcdef0123456789abcdef01234567"))
|
||||
Expect(string(data)).To(ContainSubstring("cache_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
|
||||
Expect(string(data)).To(ContainSubstring("model: owner/repo"))
|
||||
})
|
||||
|
||||
It("materializes a direct Hugging Face file reference", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
configPath := filepath.Join(modelsPath, "hf-file.yaml")
|
||||
Expect(os.WriteFile(configPath, []byte(`
|
||||
name: hf-file
|
||||
backend: transformers
|
||||
parameters:
|
||||
model: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf
|
||||
`), 0644)).To(Succeed())
|
||||
resolved := modelartifacts.Spec{
|
||||
Name: "model",
|
||||
Target: "model",
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "nomic-ai/nomic-embed-text-v1.5-GGUF", AllowPatterns: []string{"nomic-embed-text-v1.5.f16.gguf"}, Revision: "main"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
},
|
||||
}
|
||||
fake := &preloadArtifactMaterializer{result: modelartifacts.Result{
|
||||
Spec: resolved,
|
||||
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
|
||||
}, seen: make(chan modelartifacts.Spec, 1)}
|
||||
loader := NewModelConfigLoader(modelsPath, WithArtifactMaterializer(fake))
|
||||
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
|
||||
Expect(loader.PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
|
||||
|
||||
loaded, found := loader.GetModelConfig("hf-file")
|
||||
Expect(found).To(BeTrue())
|
||||
Expect(loaded.Model).To(Equal("https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf"))
|
||||
Expect(loaded.ModelFileName()).To(Equal(fake.result.RelativePath))
|
||||
Expect(fake.seen).To(Receive(SatisfyAll(
|
||||
WithTransform(func(spec modelartifacts.Spec) string { return spec.Source.Repo }, Equal("nomic-ai/nomic-embed-text-v1.5-GGUF")),
|
||||
WithTransform(func(spec modelartifacts.Spec) []string { return spec.Source.AllowPatterns }, Equal([]string{"nomic-embed-text-v1.5.f16.gguf"})),
|
||||
)))
|
||||
})
|
||||
|
||||
It("falls back to the legacy path when inferred materialization fails", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
configPath := filepath.Join(modelsPath, "hf-legacy.yaml")
|
||||
Expect(os.WriteFile(configPath, []byte(`
|
||||
name: hf-legacy
|
||||
backend: transformers
|
||||
parameters:
|
||||
model: https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf
|
||||
`), 0644)).To(Succeed())
|
||||
fake := &preloadArtifactMaterializer{err: context.Canceled, seen: make(chan modelartifacts.Spec, 1)}
|
||||
loader := NewModelConfigLoader(modelsPath, WithArtifactMaterializer(fake))
|
||||
Expect(loader.LoadModelConfigsFromPath(modelsPath)).To(Succeed())
|
||||
Expect(loader.PreloadWithContext(context.Background(), modelsPath)).To(Succeed())
|
||||
|
||||
loaded, found := loader.GetModelConfig("hf-legacy")
|
||||
Expect(found).To(BeTrue())
|
||||
Expect(loaded.Artifacts).To(BeEmpty())
|
||||
Expect(loaded.Model).To(Equal("https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf"))
|
||||
Expect(fake.seen).To(Receive())
|
||||
})
|
||||
|
||||
It("does not hold the loader lock while materialization blocks", func() {
|
||||
seen := make(chan modelartifacts.Spec, 1)
|
||||
release := make(chan struct{})
|
||||
fake := &preloadArtifactMaterializer{seen: seen, release: release}
|
||||
loader := NewModelConfigLoader(GinkgoT().TempDir(), WithArtifactMaterializer(fake))
|
||||
loader.Lock()
|
||||
loader.configs["managed"] = ModelConfig{
|
||||
Name: "managed",
|
||||
Artifacts: []modelartifacts.Spec{{
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
|
||||
}},
|
||||
}
|
||||
loader.Unlock()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- loader.PreloadWithContext(context.Background(), loader.modelPath) }()
|
||||
<-seen
|
||||
lookupDone := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = loader.GetModelConfig("managed")
|
||||
close(lookupDone)
|
||||
}()
|
||||
Eventually(lookupDone).Should(BeClosed())
|
||||
close(release)
|
||||
Expect(<-done).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("propagates preload cancellation", func() {
|
||||
seen := make(chan modelartifacts.Spec, 1)
|
||||
release := make(chan struct{})
|
||||
fake := &preloadArtifactMaterializer{seen: seen, release: release}
|
||||
loader := NewModelConfigLoader(GinkgoT().TempDir(), WithArtifactMaterializer(fake))
|
||||
loader.Lock()
|
||||
loader.configs["managed"] = ModelConfig{
|
||||
Name: "managed",
|
||||
Artifacts: []modelartifacts.Spec{{
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
|
||||
}},
|
||||
}
|
||||
loader.Unlock()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- loader.PreloadWithContext(ctx, loader.modelPath) }()
|
||||
<-seen
|
||||
cancel()
|
||||
Expect(<-done).To(MatchError(context.Canceled))
|
||||
})
|
||||
|
||||
It("does not overwrite a config changed during materialization", func() {
|
||||
seen := make(chan modelartifacts.Spec, 1)
|
||||
release := make(chan struct{})
|
||||
fake := &preloadArtifactMaterializer{
|
||||
seen: seen, release: release,
|
||||
result: modelartifacts.Result{RelativePath: ".artifacts/huggingface/cached/snapshot"},
|
||||
}
|
||||
loader := NewModelConfigLoader(GinkgoT().TempDir(), WithArtifactMaterializer(fake))
|
||||
loader.Lock()
|
||||
loader.configs["managed"] = ModelConfig{
|
||||
Name: "managed", Description: "before",
|
||||
Artifacts: []modelartifacts.Spec{{
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
|
||||
}},
|
||||
}
|
||||
loader.Unlock()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- loader.PreloadWithContext(context.Background(), loader.modelPath) }()
|
||||
<-seen
|
||||
loader.UpdateModelConfig("managed", func(cfg *ModelConfig) {
|
||||
cfg.Description = "changed concurrently"
|
||||
})
|
||||
close(release)
|
||||
Expect(<-done).NotTo(HaveOccurred())
|
||||
loaded, found := loader.GetModelConfig("managed")
|
||||
Expect(found).To(BeTrue())
|
||||
Expect(loaded.Description).To(Equal("changed concurrently"))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("ModelConfigLoader.GetModelsConflictingWith", func() {
|
||||
var bcl *ModelConfigLoader
|
||||
|
||||
|
||||
@@ -4,9 +4,12 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -28,6 +31,40 @@ var _ = Describe("Test cases for config related functions", func() {
|
||||
})
|
||||
})
|
||||
|
||||
It("round-trips and validates a managed model artifact", func() {
|
||||
raw := []byte(`
|
||||
name: qwen-asr
|
||||
backend: qwen-asr
|
||||
artifacts:
|
||||
- name: model
|
||||
target: model
|
||||
source:
|
||||
type: huggingface
|
||||
repo: Qwen/Qwen3-ASR-1.7B
|
||||
parameters:
|
||||
model: Qwen/Qwen3-ASR-1.7B
|
||||
`)
|
||||
var cfg ModelConfig
|
||||
Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed())
|
||||
Expect(cfg.Artifacts).To(HaveLen(1))
|
||||
valid, err := cfg.Validate()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(valid).To(BeTrue())
|
||||
})
|
||||
|
||||
It("derives a managed snapshot filename without replacing the logical model", func() {
|
||||
const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
cfg := ModelConfig{
|
||||
Artifacts: []modelartifacts.Spec{{
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
|
||||
Resolved: &modelartifacts.Resolved{CacheKey: cacheKey},
|
||||
}},
|
||||
}
|
||||
cfg.Model = "owner/repo"
|
||||
Expect(cfg.Model).To(Equal("owner/repo"))
|
||||
Expect(cfg.ModelFileName()).To(Equal(filepath.Join(".artifacts", "huggingface", cacheKey, "snapshot")))
|
||||
})
|
||||
|
||||
Context("Test Read configuration functions", func() {
|
||||
It("Test Validate", func() {
|
||||
tmp, err := os.CreateTemp("", "config.yaml")
|
||||
@@ -834,4 +871,17 @@ var _ = Describe("ModelConfig alias", func() {
|
||||
Expect(ok).To(BeFalse())
|
||||
Expect(err).To(MatchError(ContainSubstring("pure redirect")))
|
||||
})
|
||||
|
||||
It("rejects artifacts on alias configurations", func() {
|
||||
cfg := ModelConfig{
|
||||
Name: "alias-name",
|
||||
Alias: "target-name",
|
||||
Artifacts: []modelartifacts.Spec{{
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
|
||||
}},
|
||||
}
|
||||
valid, err := cfg.Validate()
|
||||
Expect(valid).To(BeFalse())
|
||||
Expect(err).To(MatchError(ContainSubstring("alias")))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -4,10 +4,57 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var managedArtifactBackends = map[string]struct{}{
|
||||
"transformers": {}, "huggingface-embeddings": {}, "sentencetransformers": {},
|
||||
"transformers-musicgen": {}, "mamba": {}, "diffusers": {}, "qwen-asr": {},
|
||||
"fish-speech": {}, "nemo": {}, "voxcpm": {}, "qwen-tts": {},
|
||||
"liquid-audio": {}, "vllm": {}, "vllm-omni": {}, "sglang": {},
|
||||
}
|
||||
|
||||
// AttachPrimaryArtifact adds the controller-managed source only when the
|
||||
// importer selected the same repository and a migrated backend.
|
||||
func AttachPrimaryArtifact(model gallery.ModelConfig, details Details) (gallery.ModelConfig, error) {
|
||||
if len(model.Files) != 0 || details.HuggingFace == nil || details.HuggingFace.ModelID == "" {
|
||||
return model, nil
|
||||
}
|
||||
var cfg config.ModelConfig
|
||||
if err := yaml.Unmarshal([]byte(model.ConfigFile), &cfg); err != nil {
|
||||
return gallery.ModelConfig{}, err
|
||||
}
|
||||
if _, supported := managedArtifactBackends[cfg.Backend]; !supported {
|
||||
return model, nil
|
||||
}
|
||||
if len(cfg.Artifacts) != 0 || cfg.Model != details.HuggingFace.ModelID {
|
||||
return model, nil
|
||||
}
|
||||
var document map[string]any
|
||||
if err := yaml.Unmarshal([]byte(model.ConfigFile), &document); err != nil {
|
||||
return gallery.ModelConfig{}, err
|
||||
}
|
||||
document["artifacts"] = []map[string]any{{
|
||||
"name": modelartifacts.TargetModel,
|
||||
"target": modelartifacts.TargetModel,
|
||||
"source": map[string]any{
|
||||
"type": modelartifacts.SourceTypeHuggingFace,
|
||||
"repo": details.HuggingFace.ModelID,
|
||||
},
|
||||
}}
|
||||
encoded, err := yaml.Marshal(document)
|
||||
if err != nil {
|
||||
return gallery.ModelConfig{}, err
|
||||
}
|
||||
model.ConfigFile = string(encoded)
|
||||
return model, nil
|
||||
}
|
||||
|
||||
// LocalModelPath normalizes a model URI for backends that treat the model
|
||||
// field as a HuggingFace repo id or local filesystem path (mlx, mlx-vlm,
|
||||
// vllm, transformers, diffusers). A "file://" import URI is reduced to the
|
||||
|
||||
@@ -4,8 +4,11 @@ import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/core/gallery/importers"
|
||||
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var _ = Describe("importer helpers", func() {
|
||||
@@ -104,3 +107,50 @@ var _ = Describe("importer helpers", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("managed artifact attachment", func() {
|
||||
details := importers.Details{
|
||||
URI: "https://huggingface.co/owner/repo",
|
||||
HuggingFace: &hfapi.ModelDetails{ModelID: "owner/repo"},
|
||||
}
|
||||
|
||||
It("attaches a source when the imported model is the discovered repository", func() {
|
||||
input := gallery.ModelConfig{ConfigFile: "backend: transformers\nparameters:\n model: owner/repo\n"}
|
||||
output, err := importers.AttachPrimaryArtifact(input, details)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var cfg config.ModelConfig
|
||||
Expect(yaml.Unmarshal([]byte(output.ConfigFile), &cfg)).To(Succeed())
|
||||
Expect(cfg.Artifacts).To(HaveLen(1))
|
||||
Expect(cfg.Artifacts[0].Source.Repo).To(Equal("owner/repo"))
|
||||
})
|
||||
|
||||
It("preserves explicit gallery files", func() {
|
||||
input := gallery.ModelConfig{
|
||||
ConfigFile: "parameters:\n model: owner/repo\n",
|
||||
Files: []gallery.File{{Filename: "model.gguf", URI: "https://huggingface.co/owner/repo/resolve/main/model.gguf"}},
|
||||
}
|
||||
output, err := importers.AttachPrimaryArtifact(input, details)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(output.ConfigFile).To(Equal(input.ConfigFile))
|
||||
})
|
||||
|
||||
It("does not attach a managed source to an unmigrated backend", func() {
|
||||
input := gallery.ModelConfig{ConfigFile: "backend: custom-python\nparameters:\n model: owner/repo\n"}
|
||||
output, err := importers.AttachPrimaryArtifact(input, details)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(output.ConfigFile).To(Equal(input.ConfigFile))
|
||||
})
|
||||
|
||||
DescribeTable("does not guess an unrelated or local source",
|
||||
func(model string, candidate importers.Details) {
|
||||
input := gallery.ModelConfig{ConfigFile: "parameters:\n model: " + model + "\n"}
|
||||
output, err := importers.AttachPrimaryArtifact(input, candidate)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var cfg config.ModelConfig
|
||||
Expect(yaml.Unmarshal([]byte(output.ConfigFile), &cfg)).To(Succeed())
|
||||
Expect(cfg.Artifacts).To(BeEmpty())
|
||||
},
|
||||
Entry("different repo", "other/repo", details),
|
||||
Entry("local file", "/models/repo", importers.Details{URI: "file:///models/repo"}),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -318,6 +318,10 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
modelConfig, err = AttachPrimaryArtifact(modelConfig, details)
|
||||
if err != nil {
|
||||
return gallery.ModelConfig{}, fmt.Errorf("attach managed artifact: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
94
core/gallery/model_artifacts.go
Normal file
94
core/gallery/model_artifacts.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package gallery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
|
||||
type ArtifactMaterializer interface {
|
||||
Ensure(context.Context, string, modelartifacts.Spec) (modelartifacts.Result, error)
|
||||
}
|
||||
|
||||
type installOptions struct {
|
||||
materializer ArtifactMaterializer
|
||||
}
|
||||
|
||||
type InstallOption func(*installOptions)
|
||||
|
||||
func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption {
|
||||
return func(options *installOptions) {
|
||||
if materializer != nil {
|
||||
options.materializer = materializer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyInstallOptions(options ...InstallOption) installOptions {
|
||||
result := installOptions{materializer: modelartifacts.NewDefaultManager()}
|
||||
for _, option := range options {
|
||||
option(&result)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func bindPrimaryArtifact(ctx context.Context, modelsPath string, typed *config.ModelConfig, configMap map[string]any, materializer ArtifactMaterializer, artifactSpec modelartifacts.Spec, inferred bool) (bool, error) {
|
||||
result, err := materializer.Ensure(ctx, modelsPath, artifactSpec)
|
||||
if err != nil {
|
||||
if inferred {
|
||||
xlog.Warn("falling back to legacy model loading after artifact materialization failed", "model", typed.Name, "error", err)
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("materialize primary model artifact: %w", err)
|
||||
}
|
||||
next := []modelartifacts.Spec{result.Spec}
|
||||
if len(typed.Artifacts) > 1 {
|
||||
next = append(next, typed.Artifacts[1:]...)
|
||||
}
|
||||
typed.Artifacts = next
|
||||
artifactYAML, err := yaml.Marshal(typed.Artifacts)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var artifactValue any
|
||||
if err := yaml.Unmarshal(artifactYAML, &artifactValue); err != nil {
|
||||
return false, err
|
||||
}
|
||||
configMap["artifacts"] = artifactValue
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func writeModelConfigAtomic(fileName string, data []byte) error {
|
||||
temporary, err := os.CreateTemp(filepath.Dir(fileName), ".model-config-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryName := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryName) }()
|
||||
if err := temporary.Chmod(0600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(data); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(temporaryName, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temporaryName, fileName)
|
||||
}
|
||||
240
core/gallery/model_artifacts_test.go
Normal file
240
core/gallery/model_artifacts_test.go
Normal file
@@ -0,0 +1,240 @@
|
||||
package gallery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
)
|
||||
|
||||
type fakeArtifactMaterializer struct {
|
||||
result modelartifacts.Result
|
||||
err error
|
||||
seen []modelartifacts.Spec
|
||||
}
|
||||
|
||||
func (f *fakeArtifactMaterializer) Ensure(_ context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
|
||||
f.seen = append(f.seen, spec)
|
||||
return f.result, f.err
|
||||
}
|
||||
|
||||
var _ = Describe("gallery artifact installation", func() {
|
||||
It("persists resolved state only after materialization succeeds", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
resolved := modelartifacts.Spec{
|
||||
Name: "model", Target: "model",
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
},
|
||||
}
|
||||
fake := &fakeArtifactMaterializer{result: modelartifacts.Result{
|
||||
Spec: resolved,
|
||||
RelativePath: ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot",
|
||||
}}
|
||||
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
|
||||
backend: transformers
|
||||
artifacts:
|
||||
- name: model
|
||||
target: model
|
||||
source:
|
||||
type: huggingface
|
||||
repo: owner/repo
|
||||
parameters:
|
||||
model: owner/repo
|
||||
unknown_extension:
|
||||
keep: true
|
||||
`}
|
||||
|
||||
installed, err := gallery.InstallModel(context.Background(), state, "managed", definition, nil, nil, false,
|
||||
gallery.WithArtifactMaterializer(fake))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(fake.seen).To(HaveLen(1))
|
||||
Expect(installed.Model).To(Equal("owner/repo"))
|
||||
Expect(installed.ModelFileName()).To(Equal(fake.result.RelativePath))
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(modelsPath, "managed.yaml"))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
var persisted map[string]any
|
||||
Expect(yaml.Unmarshal(data, &persisted)).To(Succeed())
|
||||
Expect(persisted).To(HaveKey("unknown_extension"))
|
||||
parameters := persisted["parameters"].(map[string]any)
|
||||
Expect(parameters["model"]).To(Equal("owner/repo"))
|
||||
artifacts := persisted["artifacts"].([]any)
|
||||
resolvedMap := artifacts[0].(map[string]any)["resolved"].(map[string]any)
|
||||
Expect(resolvedMap["revision"]).To(Equal(resolved.Resolved.Revision))
|
||||
})
|
||||
|
||||
It("does not replace an existing config when materialization fails", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
configPath := filepath.Join(modelsPath, "managed.yaml")
|
||||
Expect(os.WriteFile(configPath, []byte("name: old\n"), 0644)).To(Succeed())
|
||||
fake := &fakeArtifactMaterializer{err: errors.New("acquisition failed")}
|
||||
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
|
||||
artifacts:
|
||||
- source: {type: huggingface, repo: owner/repo}
|
||||
parameters: {model: owner/repo}
|
||||
`}
|
||||
_, err = gallery.InstallModel(context.Background(), state, "managed", definition, nil, nil, false,
|
||||
gallery.WithArtifactMaterializer(fake))
|
||||
Expect(err).To(MatchError(ContainSubstring("acquisition failed")))
|
||||
Expect(os.ReadFile(configPath)).To(Equal([]byte("name: old\n")))
|
||||
})
|
||||
|
||||
It("does not replace an existing config when a legacy file download fails", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
configPath := filepath.Join(modelsPath, "legacy.yaml")
|
||||
Expect(os.WriteFile(configPath, []byte("name: old\n"), 0644)).To(Succeed())
|
||||
definition := &gallery.ModelConfig{
|
||||
Name: "legacy",
|
||||
ConfigFile: "parameters: {model: owner/legacy}\n",
|
||||
Files: []gallery.File{{
|
||||
Filename: "missing.bin",
|
||||
URI: "file:///definitely-not-a-localai-test-file",
|
||||
}},
|
||||
}
|
||||
|
||||
_, err = gallery.InstallModel(context.Background(), state, "legacy", definition, nil, nil, false)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(os.ReadFile(configPath)).To(Equal([]byte("name: old\n")))
|
||||
})
|
||||
|
||||
It("blocks an explicitly unsafe repository before materialization", func() {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
Expect(r.URL.Path).To(Equal("/api/models/owner/repo/scan"))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := w.Write([]byte(`{"hasUnsafeFile":true}`))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}))
|
||||
DeferCleanup(server.Close)
|
||||
originalEndpoint := downloader.HF_ENDPOINT
|
||||
downloader.HF_ENDPOINT = server.URL
|
||||
DeferCleanup(func() { downloader.HF_ENDPOINT = originalEndpoint })
|
||||
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
fake := &fakeArtifactMaterializer{err: errors.New("unsafe install must not materialize")}
|
||||
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
|
||||
artifacts:
|
||||
- source: {type: huggingface, repo: owner/repo}
|
||||
parameters: {model: owner/repo}
|
||||
`}
|
||||
|
||||
_, err = gallery.InstallModel(context.Background(), state, "managed", definition, nil, nil, true,
|
||||
gallery.WithArtifactMaterializer(fake))
|
||||
Expect(err).To(MatchError(downloader.ErrUnsafeFilesFound))
|
||||
Expect(fake.seen).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("falls back to the legacy path when inferred materialization fails", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
fake := &fakeArtifactMaterializer{err: errors.New("legacy install must not materialize")}
|
||||
definition := &gallery.ModelConfig{Name: "legacy", ConfigFile: `
|
||||
backend: transformers
|
||||
parameters:
|
||||
model: owner/legacy
|
||||
`}
|
||||
installed, err := gallery.InstallModel(
|
||||
context.Background(), state, "legacy", definition, nil, nil, false,
|
||||
gallery.WithArtifactMaterializer(fake),
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(fake.seen).To(HaveLen(1))
|
||||
Expect(installed.Model).To(Equal("owner/legacy"))
|
||||
Expect(installed.Artifacts).To(BeEmpty())
|
||||
Expect(filepath.Join(modelsPath, "legacy.yaml")).To(BeAnExistingFile())
|
||||
})
|
||||
|
||||
It("passes the controller materializer through named gallery installs", func() {
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
definitionPath := filepath.Join(modelsPath, "definition.yaml")
|
||||
definition, err := yaml.Marshal(gallery.ModelConfig{Name: "managed", ConfigFile: `
|
||||
backend: transformers
|
||||
artifacts:
|
||||
- source: {type: huggingface, repo: owner/repo}
|
||||
parameters: {model: owner/repo}
|
||||
`})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(os.WriteFile(definitionPath, definition, 0644)).To(Succeed())
|
||||
galleryPath := filepath.Join(modelsPath, "index.yaml")
|
||||
index, err := yaml.Marshal([]gallery.GalleryModel{{Metadata: gallery.Metadata{
|
||||
Name: "managed",
|
||||
URL: "file://" + definitionPath,
|
||||
}}})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(os.WriteFile(galleryPath, index, 0644)).To(Succeed())
|
||||
galleries := []config.Gallery{{Name: "test", URL: "file://" + galleryPath}}
|
||||
fake := &fakeArtifactMaterializer{result: modelartifacts.Result{Spec: modelartifacts.Spec{
|
||||
Name: "model", Target: "model",
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
},
|
||||
}}}
|
||||
|
||||
Expect(gallery.InstallModelFromGallery(
|
||||
context.Background(), galleries, nil, state, nil, "test@managed",
|
||||
gallery.GalleryModel{}, nil, false, false, false,
|
||||
gallery.WithArtifactMaterializer(fake),
|
||||
)).To(Succeed())
|
||||
Expect(fake.seen).To(HaveLen(1))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("artifact cache deletion policy", func() {
|
||||
It("deletes the installed config without treating its snapshot directory as a regular file", func() {
|
||||
const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
state, err := system.GetSystemState(system.WithModelPath(modelsPath))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
artifactRoot := filepath.Join(modelsPath, ".artifacts", "huggingface", cacheKey)
|
||||
Expect(os.MkdirAll(filepath.Join(artifactRoot, "snapshot"), 0750)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artifactRoot, "manifest.json"), []byte("{}"), 0644)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(modelsPath, "managed.yaml"), []byte(`
|
||||
name: managed
|
||||
artifacts:
|
||||
- name: model
|
||||
target: model
|
||||
source: {type: huggingface, repo: owner/repo}
|
||||
resolved:
|
||||
endpoint: https://huggingface.co
|
||||
revision: 0123456789abcdef0123456789abcdef01234567
|
||||
cache_key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
||||
parameters:
|
||||
model: owner/repo
|
||||
`), 0644)).To(Succeed())
|
||||
|
||||
Expect(gallery.DeleteModelFromSystem(state, "managed")).To(Succeed())
|
||||
Expect(filepath.Join(modelsPath, "managed.yaml")).NotTo(BeAnExistingFile())
|
||||
Expect(artifactRoot).To(BeADirectory())
|
||||
Expect(filepath.Join(artifactRoot, "snapshot")).To(BeADirectory())
|
||||
Expect(filepath.Join(artifactRoot, "manifest.json")).To(BeAnExistingFile())
|
||||
})
|
||||
})
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
lconfig "github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
"github.com/mudler/LocalAI/pkg/utils"
|
||||
|
||||
@@ -77,7 +79,7 @@ func InstallModelFromGallery(
|
||||
modelGalleries, backendGalleries []lconfig.Gallery,
|
||||
systemState *system.SystemState,
|
||||
modelLoader *model.ModelLoader,
|
||||
name string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend, requireBackendIntegrity bool) error {
|
||||
name string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend, requireBackendIntegrity bool, options ...InstallOption) error {
|
||||
|
||||
applyModel := func(model *GalleryModel) error {
|
||||
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
|
||||
@@ -129,7 +131,7 @@ func InstallModelFromGallery(
|
||||
}
|
||||
}
|
||||
|
||||
installedModel, err := InstallModel(ctx, systemState, installName, &config, model.Overrides, downloadStatus, enforceScan)
|
||||
installedModel, err := InstallModel(ctx, systemState, installName, &config, model.Overrides, downloadStatus, enforceScan, options...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -158,7 +160,9 @@ func InstallModelFromGallery(
|
||||
return applyModel(model)
|
||||
}
|
||||
|
||||
func InstallModel(ctx context.Context, systemState *system.SystemState, nameOverride string, config *ModelConfig, configOverrides map[string]any, downloadStatus func(string, string, string, float64), enforceScan bool) (*lconfig.ModelConfig, error) {
|
||||
func InstallModel(ctx context.Context, systemState *system.SystemState, nameOverride string, galleryConfig *ModelConfig, configOverrides map[string]any, downloadStatus func(string, string, string, float64), enforceScan bool, options ...InstallOption) (*lconfig.ModelConfig, error) {
|
||||
installOptions := applyInstallOptions(options...)
|
||||
config := galleryConfig
|
||||
basePath := systemState.Model.ModelsPath
|
||||
// Create base path if it doesn't exist
|
||||
err := os.MkdirAll(basePath, 0750)
|
||||
@@ -170,59 +174,6 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
||||
xlog.Debug("Config overrides", "overrides", configOverrides)
|
||||
}
|
||||
|
||||
// Download files and verify their SHA
|
||||
for i, file := range config.Files {
|
||||
// Check for cancellation before each file
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
|
||||
|
||||
if err := utils.VerifyPath(file.Filename, basePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create file path
|
||||
filePath := filepath.Join(basePath, file.Filename)
|
||||
|
||||
if enforceScan {
|
||||
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
|
||||
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
||||
xlog.Error("Contains unsafe file(s)!", "model", config.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
uri := downloader.URI(file.URI)
|
||||
if err := uri.DownloadFileWithContext(ctx, filePath, file.SHA256, i, len(config.Files), downloadStatus); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Write prompt template contents to separate files
|
||||
for _, template := range config.PromptTemplates {
|
||||
if err := utils.VerifyPath(template.Name+".tmpl", basePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Create file path
|
||||
filePath := filepath.Join(basePath, template.Name+".tmpl")
|
||||
|
||||
// Create parent directory
|
||||
err := os.MkdirAll(filepath.Dir(filePath), 0750)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create parent directory for prompt template %q: %v", template.Name, err)
|
||||
}
|
||||
// Create and write file content
|
||||
err = os.WriteFile(filePath, []byte(template.Content), 0644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
|
||||
}
|
||||
|
||||
xlog.Debug("Prompt template written", "template", template.Name)
|
||||
}
|
||||
|
||||
name := config.Name
|
||||
if nameOverride != "" {
|
||||
name = nameOverride
|
||||
@@ -233,11 +184,11 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
||||
}
|
||||
|
||||
modelConfig := lconfig.ModelConfig{}
|
||||
configFilePath := filepath.Join(basePath, name+".yaml")
|
||||
var updatedConfigYAML []byte
|
||||
writeConfig := len(configOverrides) != 0 || len(config.ConfigFile) != 0
|
||||
|
||||
// write config file
|
||||
if len(configOverrides) != 0 || len(config.ConfigFile) != 0 {
|
||||
configFilePath := filepath.Join(basePath, name+".yaml")
|
||||
|
||||
if writeConfig {
|
||||
// Read and update config file as map[string]interface{}
|
||||
configMap := make(map[string]any)
|
||||
err = yaml.Unmarshal([]byte(config.ConfigFile), &configMap)
|
||||
@@ -253,8 +204,8 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
||||
}
|
||||
}
|
||||
|
||||
// Write updated config file
|
||||
updatedConfigYAML, err := yaml.Marshal(configMap)
|
||||
// Marshal the merged map for typed validation and default application.
|
||||
updatedConfigYAML, err = yaml.Marshal(configMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal updated config YAML: %v", err)
|
||||
}
|
||||
@@ -301,19 +252,110 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
||||
}
|
||||
}
|
||||
|
||||
// Re-marshal from configMap to preserve unknown fields
|
||||
updatedConfigYAML, err = yaml.Marshal(configMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal config with inference defaults: %v", err)
|
||||
}
|
||||
|
||||
if valid, err := modelConfig.Validate(); !valid {
|
||||
return nil, fmt.Errorf("failed to validate updated config YAML: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(configFilePath, updatedConfigYAML, 0644)
|
||||
if len(config.Files) == 0 {
|
||||
artifactSpec, inferred, hasArtifact, err := modelConfig.PrimaryArtifactSpec(basePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if hasArtifact && enforceScan {
|
||||
artifact, err := artifactSpec.Normalize()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
probe := downloader.URI(fmt.Sprintf("%s/%s/resolve/%s/.gitattributes", strings.TrimRight(downloader.HF_ENDPOINT, "/"), artifact.Source.Repo, url.PathEscape(artifact.Source.Revision)))
|
||||
if _, err := downloader.HuggingFaceScan(probe); errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if hasArtifact {
|
||||
applied, err := bindPrimaryArtifact(ctx, basePath, &modelConfig, configMap, installOptions.materializer, artifactSpec, inferred)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if applied {
|
||||
// Re-marshal from configMap after artifact binding to preserve unknown fields.
|
||||
updatedConfigYAML, err = yaml.Marshal(configMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal config with inference defaults: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Download files and verify their SHA
|
||||
tasks := make([]downloader.FileTask, 0, len(config.Files))
|
||||
for i, file := range config.Files {
|
||||
// Check for cancellation before each file
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
|
||||
|
||||
if err := utils.VerifyPath(file.Filename, basePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create file path
|
||||
filePath := filepath.Join(basePath, file.Filename)
|
||||
|
||||
if enforceScan {
|
||||
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
|
||||
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
||||
xlog.Error("Contains unsafe file(s)!", "model", config.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
tasks = append(tasks, downloader.FileTask{
|
||||
URI: downloader.URI(file.URI),
|
||||
Destination: filePath,
|
||||
SHA256: file.SHA256,
|
||||
FileIndex: i,
|
||||
TotalFiles: len(config.Files),
|
||||
})
|
||||
}
|
||||
if err := downloader.DownloadFilesWithContext(ctx, tasks, downloadStatus); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Write prompt template contents to separate files
|
||||
for _, template := range config.PromptTemplates {
|
||||
if err := utils.VerifyPath(template.Name+".tmpl", basePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Create file path
|
||||
filePath := filepath.Join(basePath, template.Name+".tmpl")
|
||||
|
||||
// Create parent directory
|
||||
err := os.MkdirAll(filepath.Dir(filePath), 0750)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to write updated config file: %v", err)
|
||||
return nil, fmt.Errorf("failed to create parent directory for prompt template %q: %v", template.Name, err)
|
||||
}
|
||||
// Create and write file content
|
||||
err = os.WriteFile(filePath, []byte(template.Content), 0644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
|
||||
}
|
||||
|
||||
xlog.Debug("Prompt template written", "template", template.Name)
|
||||
}
|
||||
|
||||
if writeConfig {
|
||||
if len(modelConfig.Artifacts) > 0 {
|
||||
modelartifacts.ReportProgress(ctx, modelartifacts.ProgressEvent{
|
||||
Phase: modelartifacts.PhasePersisting, Artifact: modelConfig.Artifacts[0].Name,
|
||||
})
|
||||
}
|
||||
if err := writeModelConfigAtomic(configFilePath, updatedConfigYAML); err != nil {
|
||||
return nil, fmt.Errorf("failed to atomically write updated config file: %w", err)
|
||||
}
|
||||
|
||||
xlog.Debug("Written config file", "file", configFilePath)
|
||||
@@ -374,7 +416,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if modelConfig.Model != "" {
|
||||
if modelConfig.Model != "" && len(modelConfig.Artifacts) == 0 {
|
||||
additionalFiles = append(additionalFiles, modelConfig.ModelFileName())
|
||||
}
|
||||
|
||||
|
||||
40
core/http/react-ui/e2e/model-artifact-operation.spec.js
Normal file
40
core/http/react-ui/e2e/model-artifact-operation.spec.js
Normal file
@@ -0,0 +1,40 @@
|
||||
import { test, expect } from './coverage-fixtures.js'
|
||||
|
||||
test('operations bar shows managed model acquisition phase and bytes', async ({ page }) => {
|
||||
await page.route('**/api/operations', (route) => route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
operations: [{
|
||||
id: 'qwen-asr',
|
||||
name: 'qwen-asr',
|
||||
fullName: 'qwen-asr',
|
||||
jobID: 'artifact-job-123',
|
||||
progress: 45,
|
||||
taskType: 'installation',
|
||||
isDeletion: false,
|
||||
isBackend: false,
|
||||
isQueued: false,
|
||||
isCancelled: false,
|
||||
cancellable: true,
|
||||
phase: 'downloading',
|
||||
currentBytes: 1073741824,
|
||||
totalBytes: 4294967296,
|
||||
}],
|
||||
}),
|
||||
}))
|
||||
let cancelledPath = ''
|
||||
await page.route('**/api/operations/artifact-job-123/cancel', (route) => {
|
||||
cancelledPath = new URL(route.request().url()).pathname
|
||||
return route.fulfill({ contentType: 'application/json', body: '{}' })
|
||||
})
|
||||
|
||||
await page.goto('/app/models')
|
||||
const operation = page.locator('.operation-item').filter({ hasText: 'qwen-asr' })
|
||||
await expect(operation).toContainText('Downloading model files')
|
||||
await expect(operation).toContainText('1 GB / 4 GB')
|
||||
await expect(operation.locator('.operation-progress')).toHaveText('45%')
|
||||
await expect(operation.locator('.operation-bar')).toHaveAttribute('style', /width: 45%/)
|
||||
|
||||
await operation.getByTitle('Cancel').click()
|
||||
expect(cancelledPath).toBe('/api/operations/artifact-job-123/cancel')
|
||||
})
|
||||
@@ -1,5 +1,14 @@
|
||||
import { useState } from 'react'
|
||||
import { useOperations } from '../hooks/useOperations'
|
||||
import { formatBytes } from '../utils/format'
|
||||
|
||||
const artifactPhaseLabels = {
|
||||
resolving: 'Resolving model files',
|
||||
downloading: 'Downloading model files',
|
||||
verifying: 'Verifying model files',
|
||||
committing: 'Finalizing model installation',
|
||||
persisting: 'Saving model configuration',
|
||||
}
|
||||
|
||||
const nodeStatusLabels = {
|
||||
success: 'Done',
|
||||
@@ -26,6 +35,10 @@ export default function OperationsBar() {
|
||||
const nodes = Array.isArray(op.nodes) ? op.nodes : []
|
||||
const canExpand = nodes.length > 1
|
||||
const isOpen = !!expanded[key]
|
||||
const phaseLabel = artifactPhaseLabels[op.phase]
|
||||
const byteLabel = Number.isFinite(op.currentBytes) && Number.isFinite(op.totalBytes) && op.totalBytes > 0
|
||||
? `${formatBytes(op.currentBytes)} / ${formatBytes(op.totalBytes)}`
|
||||
: ''
|
||||
return (
|
||||
<div key={key} className="operation-item">
|
||||
<div className="operation-info">
|
||||
@@ -68,7 +81,17 @@ export default function OperationsBar() {
|
||||
Cancelling...
|
||||
</span>
|
||||
)}
|
||||
{!op.error && op.message && !op.isQueued && !op.isCancelled && (
|
||||
{!op.error && phaseLabel && !op.isCancelled && (
|
||||
<span className="operation-phase" style={{ fontSize: '0.75rem', color: 'var(--color-text-muted)', marginLeft: 'var(--spacing-xs)' }}>
|
||||
{phaseLabel}
|
||||
</span>
|
||||
)}
|
||||
{!op.error && byteLabel && !op.isCancelled && (
|
||||
<span className="operation-bytes" style={{ fontSize: '0.75rem', color: 'var(--color-text-muted)', marginLeft: 'var(--spacing-xs)' }}>
|
||||
{byteLabel}
|
||||
</span>
|
||||
)}
|
||||
{!op.error && op.message && !phaseLabel && !op.isQueued && !op.isCancelled && (
|
||||
<span style={{ fontSize: '0.75rem', color: 'var(--color-text-muted)', marginLeft: 'var(--spacing-xs)' }}>
|
||||
{op.message}
|
||||
</span>
|
||||
|
||||
@@ -173,6 +173,9 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
isCancelled := false
|
||||
isCancellable := false
|
||||
message := ""
|
||||
phase := ""
|
||||
currentBytes := int64(0)
|
||||
totalBytes := int64(0)
|
||||
|
||||
if status != nil {
|
||||
// Skip successfully completed operations
|
||||
@@ -189,6 +192,9 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
isCancelled = status.Cancelled
|
||||
isCancellable = status.Cancellable
|
||||
message = status.Message
|
||||
phase = status.Phase
|
||||
currentBytes = status.CurrentBytes
|
||||
totalBytes = status.TotalBytes
|
||||
if isDeletion {
|
||||
taskType = "deletion"
|
||||
}
|
||||
@@ -257,6 +263,15 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
if scopedNodeID != "" {
|
||||
opData["nodeID"] = scopedNodeID
|
||||
}
|
||||
if phase != "" {
|
||||
opData["phase"] = phase
|
||||
}
|
||||
if currentBytes > 0 {
|
||||
opData["currentBytes"] = currentBytes
|
||||
}
|
||||
if totalBytes > 0 {
|
||||
opData["totalBytes"] = totalBytes
|
||||
}
|
||||
if status != nil && status.Error != nil {
|
||||
opData["error"] = status.Error.Error()
|
||||
}
|
||||
@@ -909,7 +924,8 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
})
|
||||
}
|
||||
|
||||
_, err = gallery.InstallModel(context.Background(), appConfig.SystemState, model.Name, &config, model.Overrides, nil, false)
|
||||
_, err = gallery.InstallModel(context.Background(), appConfig.SystemState, model.Name, &config, model.Overrides, nil, false,
|
||||
gallery.WithArtifactMaterializer(appConfig.ModelArtifactMaterializer))
|
||||
if err != nil {
|
||||
return c.JSON(http.StatusInternalServerError, map[string]any{
|
||||
"error": err.Error(),
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/http/routes"
|
||||
"github.com/mudler/LocalAI/core/services/galleryop"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
)
|
||||
|
||||
// These specs guard the contract between the opcache (which stores
|
||||
@@ -152,4 +153,41 @@ var _ = Describe("/api/operations with node-scoped backend ops", func() {
|
||||
Expect(found).ToNot(HaveKey("nodeID"), "non-node-scoped ops must NOT carry a nodeID field")
|
||||
Expect(found["name"]).To(Equal("llama-cpp"))
|
||||
})
|
||||
|
||||
It("surfaces managed model artifact phase and byte counters", func() {
|
||||
state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
appCfg := &config.ApplicationConfig{SystemState: state}
|
||||
galleryService := galleryop.NewGalleryService(appCfg, nil)
|
||||
opcache := galleryop.NewOpCache(galleryService)
|
||||
jobID := "test-op-artifact-progress"
|
||||
opcache.Set("qwen-asr", jobID)
|
||||
galleryService.UpdateStatus(jobID, &galleryop.OpStatus{
|
||||
Phase: "downloading", CurrentBytes: 1024, TotalBytes: 4096,
|
||||
Progress: 22.5, Message: "Downloading model file: weights.safetensors", Cancellable: true,
|
||||
})
|
||||
|
||||
e := echo.New()
|
||||
routes.RegisterUIAPIRoutes(e, nil, nil, appCfg, galleryService, opcache, &application.Application{}, noopMw)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/operations", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
e.ServeHTTP(rec, req)
|
||||
Expect(rec.Code).To(Equal(http.StatusOK))
|
||||
|
||||
var envelope struct {
|
||||
Operations []map[string]any `json:"operations"`
|
||||
}
|
||||
Expect(json.Unmarshal(rec.Body.Bytes(), &envelope)).To(Succeed())
|
||||
var found map[string]any
|
||||
for _, op := range envelope.Operations {
|
||||
if op["jobID"] == jobID {
|
||||
found = op
|
||||
break
|
||||
}
|
||||
}
|
||||
Expect(found).ToNot(BeNil())
|
||||
Expect(found["phase"]).To(Equal("downloading"))
|
||||
Expect(found["currentBytes"]).To(Equal(float64(1024)))
|
||||
Expect(found["totalBytes"]).To(Equal(float64(4096)))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -25,6 +25,9 @@ type GalleryOperationRecord struct {
|
||||
OpType string `gorm:"size:32" json:"op_type"` // "model_install", "model_delete", "backend_install"
|
||||
Status string `gorm:"size:32;default:pending" json:"status"` // pending, downloading, processing, completed, failed, cancelled
|
||||
Progress float64 `json:"progress"` // 0.0 to 1.0
|
||||
Phase string `gorm:"size:32" json:"phase,omitempty"`
|
||||
CurrentBytes int64 `json:"current_bytes,omitempty"`
|
||||
TotalBytes int64 `json:"total_bytes,omitempty"`
|
||||
Message string `gorm:"type:text" json:"message,omitempty"`
|
||||
Error string `gorm:"type:text" json:"error,omitempty"`
|
||||
FileName string `gorm:"size:512" json:"file_name,omitempty"`
|
||||
@@ -84,14 +87,26 @@ func (s *GalleryStore) Create(op *GalleryOperationRecord) error {
|
||||
// op as still cancellable — otherwise the column keeps its Create-time zero
|
||||
// value (false), the UI hides the cancel button, and the orphaned op can only
|
||||
// be dismissed by waiting for the 30-minute stale reaper.
|
||||
func (s *GalleryStore) UpdateProgress(id string, progress float64, message, downloadedSize string, cancellable bool) error {
|
||||
return s.db.Model(&GalleryOperationRecord{}).Where("id = ?", id).Updates(map[string]any{
|
||||
type OperationProgressDetails struct {
|
||||
Phase string
|
||||
CurrentBytes int64
|
||||
TotalBytes int64
|
||||
}
|
||||
|
||||
func (s *GalleryStore) UpdateProgress(id string, progress float64, message, downloadedSize string, cancellable bool, details ...OperationProgressDetails) error {
|
||||
updates := map[string]any{
|
||||
"progress": progress,
|
||||
"message": message,
|
||||
"downloaded_file_size": downloadedSize,
|
||||
"cancellable": cancellable,
|
||||
"updated_at": time.Now(),
|
||||
}).Error
|
||||
}
|
||||
if len(details) > 0 {
|
||||
updates["phase"] = details[0].Phase
|
||||
updates["current_bytes"] = details[0].CurrentBytes
|
||||
updates["total_bytes"] = details[0].TotalBytes
|
||||
}
|
||||
return s.db.Model(&GalleryOperationRecord{}).Where("id = ?", id).Updates(updates).Error
|
||||
}
|
||||
|
||||
// UpdateStatus updates the status of an operation. A terminal status is never
|
||||
|
||||
65
core/services/galleryop/artifact_progress.go
Normal file
65
core/services/galleryop/artifact_progress.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package galleryop
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
type artifactProgressBridge struct {
|
||||
mu sync.Mutex
|
||||
last float64
|
||||
currentBytes int64
|
||||
totalBytes int64
|
||||
update func(*OpStatus)
|
||||
}
|
||||
|
||||
func newArtifactProgressBridge(update func(*OpStatus)) *artifactProgressBridge {
|
||||
return &artifactProgressBridge{update: update}
|
||||
}
|
||||
|
||||
func (b *artifactProgressBridge) Sink(event modelartifacts.ProgressEvent) {
|
||||
b.mu.Lock()
|
||||
progress := b.last
|
||||
message := "Preparing model files"
|
||||
switch event.Phase {
|
||||
case modelartifacts.PhaseResolving:
|
||||
progress = max(progress, 0)
|
||||
message = "Resolving model files"
|
||||
case modelartifacts.PhaseDownloading:
|
||||
if event.TotalBytes > 0 {
|
||||
progress = max(progress, min(90, float64(event.CurrentBytes)*90/float64(event.TotalBytes)))
|
||||
}
|
||||
message = fmt.Sprintf("Downloading model file: %s", event.File)
|
||||
case modelartifacts.PhaseVerifying:
|
||||
progress = max(progress, 95)
|
||||
message = "Verifying model files"
|
||||
case modelartifacts.PhaseCommitting:
|
||||
progress = max(progress, 99)
|
||||
message = "Finalizing model installation"
|
||||
case modelartifacts.PhasePersisting:
|
||||
progress = max(progress, 99)
|
||||
message = "Saving model configuration"
|
||||
}
|
||||
b.last = progress
|
||||
b.currentBytes = max(b.currentBytes, event.CurrentBytes)
|
||||
b.totalBytes = max(b.totalBytes, event.TotalBytes)
|
||||
status := &OpStatus{
|
||||
Phase: string(event.Phase), Message: message, FileName: event.File,
|
||||
Progress: progress, CurrentBytes: b.currentBytes, TotalBytes: b.totalBytes,
|
||||
Cancellable: true,
|
||||
}
|
||||
update := b.update
|
||||
b.mu.Unlock()
|
||||
if update != nil {
|
||||
update(status)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *artifactProgressBridge) ClampLegacy(progress float64) float64 {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.last = max(b.last, progress)
|
||||
return b.last
|
||||
}
|
||||
33
core/services/galleryop/artifact_progress_test.go
Normal file
33
core/services/galleryop/artifact_progress_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package galleryop
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
var _ = Describe("artifact operation progress", func() {
|
||||
It("maps phases and never moves percentage backward", func() {
|
||||
var statuses []*OpStatus
|
||||
bridge := newArtifactProgressBridge(func(status *OpStatus) {
|
||||
copy := *status
|
||||
statuses = append(statuses, ©)
|
||||
})
|
||||
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseResolving})
|
||||
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseDownloading, File: "model.bin", CurrentBytes: 50, TotalBytes: 100})
|
||||
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseDownloading, File: "model.bin", CurrentBytes: 10, TotalBytes: 100})
|
||||
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseVerifying, CurrentBytes: 100, TotalBytes: 100})
|
||||
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhaseCommitting, CurrentBytes: 100, TotalBytes: 100})
|
||||
bridge.Sink(modelartifacts.ProgressEvent{Phase: modelartifacts.PhasePersisting})
|
||||
|
||||
Expect(statuses).To(HaveLen(6))
|
||||
for index := 1; index < len(statuses); index++ {
|
||||
Expect(statuses[index].Progress).To(BeNumerically(">=", statuses[index-1].Progress))
|
||||
}
|
||||
Expect(statuses[1].Phase).To(Equal("downloading"))
|
||||
Expect(statuses[1].CurrentBytes).To(Equal(int64(50)))
|
||||
Expect(statuses[5].Progress).To(Equal(float64(99)))
|
||||
Expect(statuses[5].CurrentBytes).To(Equal(int64(100)))
|
||||
})
|
||||
})
|
||||
@@ -38,9 +38,12 @@ var _ = Describe("GalleryService cancellable persistence across restart", func()
|
||||
// Simulate a progress tick: the live path always marks installs
|
||||
// cancellable while they are downloading/processing.
|
||||
svc.UpdateStatus("op-inflight", &galleryop.OpStatus{
|
||||
Message: "downloading",
|
||||
Progress: 25,
|
||||
Cancellable: true,
|
||||
Message: "downloading",
|
||||
Progress: 25,
|
||||
Phase: "downloading",
|
||||
CurrentBytes: 123,
|
||||
TotalBytes: 456,
|
||||
Cancellable: true,
|
||||
})
|
||||
|
||||
// A fresh replica boots and hydrates from the store.
|
||||
@@ -52,5 +55,8 @@ var _ = Describe("GalleryService cancellable persistence across restart", func()
|
||||
Expect(st).ToNot(BeNil(), "the in-flight op must hydrate after a restart")
|
||||
Expect(st.Cancellable).To(BeTrue(),
|
||||
"a still-active install must rehydrate as cancellable so the admin can dismiss it")
|
||||
Expect(st.Phase).To(Equal("downloading"))
|
||||
Expect(st.CurrentBytes).To(Equal(int64(123)))
|
||||
Expect(st.TotalBytes).To(Equal(int64(456)))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -122,6 +122,9 @@ var _ = Describe("OpStatus JSON wire format", func() {
|
||||
original := &galleryop.OpStatus{
|
||||
Progress: 42.0,
|
||||
Message: "downloading",
|
||||
Phase: "downloading",
|
||||
CurrentBytes: 123,
|
||||
TotalBytes: 456,
|
||||
GalleryElementName: "vllm",
|
||||
Error: errors.New("disk full"),
|
||||
Processed: true,
|
||||
@@ -134,6 +137,9 @@ var _ = Describe("OpStatus JSON wire format", func() {
|
||||
Expect(got.Error).ToNot(BeNil(), "the error must survive the round-trip — peer replicas need to surface the failure")
|
||||
Expect(got.Error.Error()).To(Equal("disk full"))
|
||||
Expect(got.Progress).To(Equal(42.0))
|
||||
Expect(got.Phase).To(Equal("downloading"))
|
||||
Expect(got.CurrentBytes).To(Equal(int64(123)))
|
||||
Expect(got.TotalBytes).To(Equal(int64(456)))
|
||||
Expect(got.GalleryElementName).To(Equal("vllm"))
|
||||
Expect(got.Processed).To(BeTrue())
|
||||
})
|
||||
|
||||
@@ -17,6 +17,7 @@ type LocalModelManager struct {
|
||||
enforcePredownloadScans bool
|
||||
automaticallyInstallBackend bool
|
||||
requireBackendIntegrity bool
|
||||
artifactMaterializer config.ArtifactMaterializer
|
||||
}
|
||||
|
||||
// NewLocalModelManager creates a LocalModelManager from the application config.
|
||||
@@ -27,6 +28,7 @@ func NewLocalModelManager(appConfig *config.ApplicationConfig, ml *model.ModelLo
|
||||
enforcePredownloadScans: appConfig.EnforcePredownloadScans,
|
||||
automaticallyInstallBackend: appConfig.AutoloadBackendGalleries,
|
||||
requireBackendIntegrity: appConfig.RequireBackendIntegrity,
|
||||
artifactMaterializer: appConfig.ModelArtifactMaterializer,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +50,8 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g
|
||||
switch {
|
||||
case op.GalleryElement != nil:
|
||||
installedModel, err := gallery.InstallModel(ctx, m.systemState, op.GalleryElement.Name,
|
||||
op.GalleryElement, op.Req.Overrides, progressCb, m.enforcePredownloadScans)
|
||||
op.GalleryElement, op.Req.Overrides, progressCb, m.enforcePredownloadScans,
|
||||
gallery.WithArtifactMaterializer(m.artifactMaterializer))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -61,10 +64,12 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g
|
||||
case op.GalleryElementName != "":
|
||||
return gallery.InstallModelFromGallery(ctx, op.Galleries, op.BackendGalleries,
|
||||
m.systemState, m.modelLoader, op.GalleryElementName, op.Req, progressCb,
|
||||
m.enforcePredownloadScans, m.automaticallyInstallBackend, m.requireBackendIntegrity)
|
||||
m.enforcePredownloadScans, m.automaticallyInstallBackend, m.requireBackendIntegrity,
|
||||
gallery.WithArtifactMaterializer(m.artifactMaterializer))
|
||||
default:
|
||||
return installModelFromRemoteConfig(ctx, m.systemState, m.modelLoader, op.Req,
|
||||
progressCb, m.enforcePredownloadScans, m.automaticallyInstallBackend, op.BackendGalleries, m.requireBackendIntegrity)
|
||||
progressCb, m.enforcePredownloadScans, m.automaticallyInstallBackend, op.BackendGalleries, m.requireBackendIntegrity,
|
||||
gallery.WithArtifactMaterializer(m.artifactMaterializer))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
51
core/services/galleryop/model_artifact_materializer_test.go
Normal file
51
core/services/galleryop/model_artifact_materializer_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package galleryop_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/core/services/galleryop"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
)
|
||||
|
||||
type galleryOpArtifactMaterializer struct {
|
||||
seen []modelartifacts.Spec
|
||||
}
|
||||
|
||||
func (f *galleryOpArtifactMaterializer) Ensure(_ context.Context, _ string, spec modelartifacts.Spec) (modelartifacts.Result, error) {
|
||||
f.seen = append(f.seen, spec)
|
||||
spec.Resolved = &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
}
|
||||
return modelartifacts.Result{Spec: spec}, nil
|
||||
}
|
||||
|
||||
var _ = Describe("local model manager artifact materializer", func() {
|
||||
It("passes the controller materializer to direct gallery installs", func() {
|
||||
state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
materializer := &galleryOpArtifactMaterializer{}
|
||||
manager := galleryop.NewLocalModelManager(&config.ApplicationConfig{
|
||||
SystemState: state,
|
||||
ModelArtifactMaterializer: materializer,
|
||||
}, nil)
|
||||
definition := &gallery.ModelConfig{Name: "managed", ConfigFile: `
|
||||
backend: transformers
|
||||
artifacts:
|
||||
- source: {type: huggingface, repo: owner/repo}
|
||||
parameters: {model: owner/repo}
|
||||
`}
|
||||
op := &galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{
|
||||
GalleryElement: definition,
|
||||
}
|
||||
Expect(manager.InstallModel(context.Background(), op, nil)).To(Succeed())
|
||||
Expect(materializer.seen).To(HaveLen(1))
|
||||
})
|
||||
})
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/core/services/messaging"
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
"github.com/mudler/LocalAI/pkg/system"
|
||||
"github.com/mudler/LocalAI/pkg/utils"
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -52,6 +53,16 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
|
||||
|
||||
g.UpdateStatus(op.ID, &OpStatus{Message: fmt.Sprintf("processing model: %s", op.GalleryElementName), Progress: 0, Cancellable: true})
|
||||
|
||||
bridge := newArtifactProgressBridge(func(status *OpStatus) {
|
||||
status.GalleryElementName = op.GalleryElementName
|
||||
g.UpdateStatus(op.ID, status)
|
||||
})
|
||||
operationCtx := op.Context
|
||||
if operationCtx == nil {
|
||||
operationCtx = context.Background()
|
||||
}
|
||||
operationCtx = modelartifacts.WithProgressSink(operationCtx, bridge.Sink)
|
||||
|
||||
// displayDownload displays the download progress
|
||||
progressCallback := func(fileName string, current string, total string, percentage float64) {
|
||||
// Check for cancellation during progress updates
|
||||
@@ -62,6 +73,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
|
||||
default:
|
||||
}
|
||||
}
|
||||
percentage = bridge.ClampLegacy(percentage)
|
||||
g.UpdateStatus(op.ID, &OpStatus{Message: fmt.Sprintf(processingMessage, fileName, total, current), FileName: fileName, Progress: percentage, TotalFileSize: total, DownloadedFileSize: current, Cancellable: true})
|
||||
utils.DisplayDownloadFunction(fileName, current, total, percentage)
|
||||
}
|
||||
@@ -70,7 +82,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
|
||||
if op.Delete {
|
||||
err = g.modelManager.DeleteModel(op.GalleryElementName)
|
||||
} else {
|
||||
err = g.modelManager.InstallModel(op.Context, op, progressCallback)
|
||||
err = g.modelManager.InstallModel(operationCtx, op, progressCallback)
|
||||
}
|
||||
if err != nil {
|
||||
// Check if error is due to cancellation
|
||||
@@ -107,7 +119,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
|
||||
return err
|
||||
}
|
||||
|
||||
err = cl.Preload(systemState.Model.ModelsPath)
|
||||
err = cl.PreloadWithContext(operationCtx, systemState.Model.ModelsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -137,7 +149,7 @@ func (g *GalleryService) modelHandler(op *ManagementOp[gallery.GalleryModel, gal
|
||||
return nil
|
||||
}
|
||||
|
||||
func installModelFromRemoteConfig(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery, requireBackendIntegrity bool) error {
|
||||
func installModelFromRemoteConfig(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
|
||||
config, err := gallery.GetGalleryConfigFromURLWithContext[gallery.ModelConfig](ctx, req.URL, systemState.Model.ModelsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -145,7 +157,7 @@ func installModelFromRemoteConfig(ctx context.Context, systemState *system.Syste
|
||||
|
||||
config.Files = append(config.Files, req.AdditionalFiles...)
|
||||
|
||||
installedModel, err := gallery.InstallModel(ctx, systemState, req.Name, &config, req.Overrides, downloadStatus, enforceScan)
|
||||
installedModel, err := gallery.InstallModel(ctx, systemState, req.Name, &config, req.Overrides, downloadStatus, enforceScan, options...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -164,23 +176,23 @@ type galleryModel struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
func processRequests(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel, requireBackendIntegrity bool) error {
|
||||
func processRequests(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
|
||||
ctx := context.Background()
|
||||
var err error
|
||||
for _, r := range requests {
|
||||
utils.ResetDownloadTimers()
|
||||
if r.ID == "" {
|
||||
err = installModelFromRemoteConfig(ctx, systemState, modelLoader, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries, requireBackendIntegrity)
|
||||
err = installModelFromRemoteConfig(ctx, systemState, modelLoader, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries, requireBackendIntegrity, options...)
|
||||
|
||||
} else {
|
||||
err = gallery.InstallModelFromGallery(
|
||||
ctx, galleries, backendGalleries, systemState, modelLoader, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, requireBackendIntegrity)
|
||||
ctx, galleries, backendGalleries, systemState, modelLoader, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, requireBackendIntegrity, options...)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func ApplyGalleryFromFile(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool) error {
|
||||
func ApplyGalleryFromFile(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
|
||||
dat, err := os.ReadFile(s)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -191,15 +203,15 @@ func ApplyGalleryFromFile(systemState *system.SystemState, modelLoader *model.Mo
|
||||
return err
|
||||
}
|
||||
|
||||
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity)
|
||||
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity, options...)
|
||||
}
|
||||
|
||||
func ApplyGalleryFromString(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool) error {
|
||||
func ApplyGalleryFromString(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string, requireBackendIntegrity bool, options ...gallery.InstallOption) error {
|
||||
var requests []galleryModel
|
||||
err := json.Unmarshal([]byte(s), &requests)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity)
|
||||
return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests, requireBackendIntegrity, options...)
|
||||
}
|
||||
|
||||
@@ -61,6 +61,9 @@ type OpStatus struct {
|
||||
Processed bool `json:"processed"`
|
||||
Message string `json:"message"`
|
||||
Progress float64 `json:"progress"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
CurrentBytes int64 `json:"current_bytes,omitempty"`
|
||||
TotalBytes int64 `json:"total_bytes,omitempty"`
|
||||
TotalFileSize string `json:"file_size"`
|
||||
DownloadedFileSize string `json:"downloaded_size"`
|
||||
GalleryElementName string `json:"gallery_element_name"`
|
||||
@@ -89,6 +92,9 @@ type opStatusWire struct {
|
||||
Processed bool `json:"processed"`
|
||||
Message string `json:"message"`
|
||||
Progress float64 `json:"progress"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
CurrentBytes int64 `json:"current_bytes,omitempty"`
|
||||
TotalBytes int64 `json:"total_bytes,omitempty"`
|
||||
TotalFileSize string `json:"file_size"`
|
||||
DownloadedFileSize string `json:"downloaded_size"`
|
||||
GalleryElementName string `json:"gallery_element_name"`
|
||||
@@ -104,6 +110,9 @@ func (o OpStatus) MarshalJSON() ([]byte, error) {
|
||||
Processed: o.Processed,
|
||||
Message: o.Message,
|
||||
Progress: o.Progress,
|
||||
Phase: o.Phase,
|
||||
CurrentBytes: o.CurrentBytes,
|
||||
TotalBytes: o.TotalBytes,
|
||||
TotalFileSize: o.TotalFileSize,
|
||||
DownloadedFileSize: o.DownloadedFileSize,
|
||||
GalleryElementName: o.GalleryElementName,
|
||||
@@ -127,6 +136,9 @@ func (o *OpStatus) UnmarshalJSON(data []byte) error {
|
||||
o.Processed = w.Processed
|
||||
o.Message = w.Message
|
||||
o.Progress = w.Progress
|
||||
o.Phase = w.Phase
|
||||
o.CurrentBytes = w.CurrentBytes
|
||||
o.TotalBytes = w.TotalBytes
|
||||
o.TotalFileSize = w.TotalFileSize
|
||||
o.DownloadedFileSize = w.DownloadedFileSize
|
||||
o.GalleryElementName = w.GalleryElementName
|
||||
|
||||
@@ -94,6 +94,15 @@ func (g *GalleryService) BackendManager() BackendManager {
|
||||
return g.backendManager
|
||||
}
|
||||
|
||||
// ModelArtifactMaterializer returns the controller-only acquisition capability
|
||||
// used by startup paths that install gallery entries outside the operation loop.
|
||||
func (g *GalleryService) ModelArtifactMaterializer() config.ArtifactMaterializer {
|
||||
if g == nil || g.appConfig == nil {
|
||||
return nil
|
||||
}
|
||||
return g.appConfig.ModelArtifactMaterializer
|
||||
}
|
||||
|
||||
// SetNATSClient sets the NATS client for distributed progress publishing.
|
||||
// Accepting the wider MessagingClient (vs. plain Publisher) lets
|
||||
// SubscribeBroadcasts wire the wildcard subscriptions that keep peer
|
||||
@@ -167,7 +176,10 @@ func (g *GalleryService) UpdateStatus(s string, op *OpStatus) {
|
||||
xlog.Warn("Failed to persist gallery operation status", "op_id", s, "error", err)
|
||||
}
|
||||
} else {
|
||||
if err := store.UpdateProgress(s, op.Progress, op.Message, op.DownloadedFileSize, op.Cancellable); err != nil {
|
||||
if err := store.UpdateProgress(s, op.Progress, op.Message, op.DownloadedFileSize, op.Cancellable,
|
||||
distributed.OperationProgressDetails{
|
||||
Phase: op.Phase, CurrentBytes: op.CurrentBytes, TotalBytes: op.TotalBytes,
|
||||
}); err != nil {
|
||||
xlog.Warn("Failed to persist gallery operation progress", "op_id", s, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -662,6 +674,9 @@ func (g *GalleryService) Hydrate() error {
|
||||
st := &OpStatus{
|
||||
Message: op.Message,
|
||||
Progress: op.Progress,
|
||||
Phase: op.Phase,
|
||||
CurrentBytes: op.CurrentBytes,
|
||||
TotalBytes: op.TotalBytes,
|
||||
FileName: op.FileName,
|
||||
TotalFileSize: op.TotalFileSize,
|
||||
DownloadedFileSize: op.DownloadedFileSize,
|
||||
|
||||
@@ -77,4 +77,30 @@ var _ = Describe("ApplyRemoteChange", func() {
|
||||
Expect(ok1).To(BeTrue())
|
||||
Expect(ok2).To(BeTrue())
|
||||
})
|
||||
|
||||
It("loads a peer-persisted artifact binding without materializing", func() {
|
||||
const relative = ".artifacts/huggingface/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/snapshot"
|
||||
writeYAML("peer-managed", map[string]any{
|
||||
"backend": "transformers",
|
||||
"artifacts": []map[string]any{{
|
||||
"name": "model", "target": "model",
|
||||
"source": map[string]any{"type": "huggingface", "repo": "owner/repo", "revision": "main"},
|
||||
"resolved": map[string]any{
|
||||
"endpoint": "https://huggingface.co",
|
||||
"revision": "0123456789abcdef0123456789abcdef01234567",
|
||||
"cache_key": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
},
|
||||
}},
|
||||
"parameters": map[string]any{"model": "owner/repo"},
|
||||
})
|
||||
Expect(ApplyRemoteChange(loader, nil, dir, messaging.CacheInvalidateEvent{
|
||||
Element: "peer-managed", Op: "install",
|
||||
})).To(Succeed())
|
||||
loaded, found := loader.GetModelConfig("peer-managed")
|
||||
Expect(found).To(BeTrue())
|
||||
Expect(loaded.Model).To(Equal("owner/repo"))
|
||||
Expect(loaded.ModelFileName()).To(Equal(relative))
|
||||
Expect(loaded.Artifacts).To(HaveLen(1))
|
||||
Expect(loaded.Artifacts[0].Resolved.CacheKey).To(HaveLen(64))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/services/storage"
|
||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||
)
|
||||
|
||||
@@ -61,4 +63,48 @@ var _ = Describe("stageModelFiles directory models", func() {
|
||||
Expect(staged).To(ConsistOf(weights, tokenizer))
|
||||
Expect(staged).ToNot(ContainElement(modelDir))
|
||||
})
|
||||
|
||||
It("stages a content-addressed Hugging Face snapshot with its relative tree intact", func() {
|
||||
cacheKey := strings.Repeat("a", 64)
|
||||
logicalModel := "owner/repo"
|
||||
relativeSnapshot := filepath.Join(".artifacts", "huggingface", cacheKey, "snapshot")
|
||||
snapshot := filepath.Join(tmp, "models", relativeSnapshot)
|
||||
files := map[string]string{
|
||||
"config.json": "{}",
|
||||
"model.safetensors.index.json": "{}",
|
||||
"model-00001-of-00002.safetensors": "part-1",
|
||||
"model-00002-of-00002.safetensors": "part-2",
|
||||
filepath.Join("tokenizer", "vocab.json"): "{}",
|
||||
}
|
||||
for relative, contents := range files {
|
||||
path := filepath.Join(snapshot, relative)
|
||||
Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
|
||||
Expect(os.WriteFile(path, []byte(contents), 0o644)).To(Succeed())
|
||||
}
|
||||
|
||||
opts := &pb.ModelOptions{Model: logicalModel, ModelFile: snapshot}
|
||||
stagedOpts, err := router.stageModelFiles(context.Background(), node, opts, "managed-model")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
stagedPaths := make([]string, 0, len(stager.ensureCalls))
|
||||
stagedKeys := make([]string, 0, len(stager.ensureCalls))
|
||||
for _, call := range stager.ensureCalls {
|
||||
stagedPaths = append(stagedPaths, call.localPath)
|
||||
stagedKeys = append(stagedKeys, call.key)
|
||||
}
|
||||
expectedPaths := make([]string, 0, len(files))
|
||||
expectedKeys := make([]string, 0, len(files))
|
||||
for relative := range files {
|
||||
expectedPaths = append(expectedPaths, filepath.Join(snapshot, relative))
|
||||
expectedKeys = append(expectedKeys, storage.ModelKey(filepath.Join("managed-model", relative)))
|
||||
}
|
||||
Expect(stagedPaths).To(ConsistOf(expectedPaths))
|
||||
Expect(stagedKeys).To(ConsistOf(expectedKeys))
|
||||
remoteSnapshot := filepath.Join("/remote", storage.ModelKey("managed-model"))
|
||||
Expect(stagedOpts.Model).To(Equal(logicalModel))
|
||||
Expect(stagedOpts.ModelFile).To(Equal(remoteSnapshot))
|
||||
Expect(stagedOpts.ModelPath).To(Equal(remoteSnapshot))
|
||||
Expect(opts.Model).To(Equal(logicalModel))
|
||||
Expect(opts.ModelFile).To(Equal(snapshot))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -24,9 +24,13 @@ import (
|
||||
func InstallModels(ctx context.Context, galleryService *galleryop.GalleryService, galleries, backendGalleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool, downloadStatus func(string, string, string, float64), models ...string) error {
|
||||
// create an error that groups all errors
|
||||
var err error
|
||||
var installOptions []gallery.InstallOption
|
||||
if galleryService != nil {
|
||||
installOptions = append(installOptions, gallery.WithArtifactMaterializer(galleryService.ModelArtifactMaterializer()))
|
||||
}
|
||||
for _, url := range models {
|
||||
// Check if it's a model gallery, or print a warning
|
||||
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity)
|
||||
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity, installOptions...)
|
||||
if e != nil && found {
|
||||
xlog.Error("[startup] failed installing model", "error", err, "model", url)
|
||||
err = errors.Join(err, e)
|
||||
@@ -82,7 +86,7 @@ func InstallModels(ctx context.Context, galleryService *galleryop.GalleryService
|
||||
return err
|
||||
}
|
||||
|
||||
func installModel(ctx context.Context, galleries, backendGalleries []config.Gallery, modelName string, systemState *system.SystemState, modelLoader *model.ModelLoader, downloadStatus func(string, string, string, float64), enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool) (error, bool) {
|
||||
func installModel(ctx context.Context, galleries, backendGalleries []config.Gallery, modelName string, systemState *system.SystemState, modelLoader *model.ModelLoader, downloadStatus func(string, string, string, float64), enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool, options ...gallery.InstallOption) (error, bool) {
|
||||
models, err := gallery.AvailableGalleryModels(galleries, systemState)
|
||||
if err != nil {
|
||||
return err, false
|
||||
@@ -98,7 +102,7 @@ func installModel(ctx context.Context, galleries, backendGalleries []config.Gall
|
||||
}
|
||||
|
||||
xlog.Info("installing model", "model", modelName, "license", model.License)
|
||||
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity)
|
||||
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries, requireBackendIntegrity, options...)
|
||||
if err != nil {
|
||||
return err, true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user