feat: materialize Hugging Face model artifacts (#10825)

* feat(config): add model artifact source contract Assisted-by: Codex:GPT-5 [Codex] * feat(downloader): add authenticated raw-byte progress Assisted-by: Codex:GPT-5 [Codex] * feat(huggingface): resolve immutable snapshot manifests Assisted-by: Codex:GPT-5 [Codex] * feat(models): add artifact storage primitives Assisted-by: Codex:GPT-5 [Codex] * feat(models): materialize pinned Hugging Face snapshots Assisted-by: Codex:GPT-5 [Codex] * feat(models): bind managed snapshots at runtime Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): materialize model artifacts during install Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): declare managed Hugging Face artifacts Assisted-by: Codex:GPT-5 [Codex] * feat(models): preload managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * fix(gallery): retain shared artifact caches on delete Assisted-by: Codex:GPT-5 [Codex] * feat(models): report artifact acquisition progress Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load managed models from ModelFile Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load staged speech model snapshots Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): use staged snapshots in engine backends Assisted-by: Codex:GPT-5 [Codex] * test(distributed): cover staged artifact snapshots Assisted-by: Codex:GPT-5 [Codex] * docs: explain managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * docs: add product design context Assisted-by: Codex:GPT-5 [Codex] * feat(ui): show model artifact download progress Assisted-by: Codex:GPT-5 [Codex] * Eagerly materialize Hugging Face artifacts Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback. Assisted-by: Codex:GPT-5 [shell] * Refactor HF downloads through a shared executor Assisted-by: Codex:GPT-5 [shell] * drop Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
2026-08-04 04:12:22 -04:00 · 2026-07-15 01:09:33 +02:00
parent d82c38ee77
commit bcc41219f7
83 changed files with 4315 additions and 339 deletions
--- a/core/backend/llm.go
+++ b/core/backend/llm.go
@@ -109,7 +109,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 		if !slices.Contains(modelNames, modelName) {
 			utils.ResetDownloadTimers()
 			// if we failed to load the model, we try to download it
-			err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, modelName, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries, o.RequireBackendIntegrity)
+			err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, modelName, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries, o.RequireBackendIntegrity, gallery.WithArtifactMaterializer(o.ModelArtifactMaterializer))
 			if err != nil {
 				xlog.Error("failed to install model from gallery", "error", err, "model", modelFile)
 				//return nil, err
--- a/core/backend/model_artifact_options_test.go
+++ b/core/backend/model_artifact_options_test.go
@@ -0,0 +1,47 @@
+package backend
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/modelartifacts"
+)
+
+var _ = Describe("managed model runtime options", func() {
+	It("estimates weights from the committed manifest without repository fallback", func() {
+		const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+		modelsPath := GinkgoT().TempDir()
+		spec := modelartifacts.Spec{
+			Name: "model", Target: "model",
+			Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
+			Resolved: &modelartifacts.Resolved{
+				Endpoint: "https://huggingface.co",
+				Revision: "0123456789abcdef0123456789abcdef01234567",
+				CacheKey: cacheKey,
+			},
+		}
+		relative, err := modelartifacts.RelativeSnapshotPath(cacheKey)
+		Expect(err).NotTo(HaveOccurred())
+		manifest := modelartifacts.Manifest{
+			Version:  modelartifacts.ManifestVersion,
+			Artifact: spec,
+			Files: []modelartifacts.ManifestFile{{
+				Path: "weights/model.safetensors", Size: 12, SHA256: strings.Repeat("a", 64),
+			}},
+		}
+		encoded, err := json.Marshal(manifest)
+		Expect(err).NotTo(HaveOccurred())
+		manifestPath := filepath.Join(modelsPath, filepath.Dir(relative), "manifest.json")
+		Expect(os.MkdirAll(filepath.Dir(manifestPath), 0750)).To(Succeed())
+		Expect(os.WriteFile(manifestPath, encoded, 0644)).To(Succeed())
+
+		cfg := config.ModelConfig{Artifacts: []modelartifacts.Spec{spec}}
+		Expect(estimateModelSizeBytes(cfg, modelsPath)).To(Equal(int64(12)))
+	})
+})
--- a/core/backend/options.go
+++ b/core/backend/options.go
@@ -12,9 +12,10 @@ import (

 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/trace"
-	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/downloader"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/modelartifacts"
 	"github.com/mudler/LocalAI/pkg/vram"
 	"github.com/mudler/xlog"
 )
@@ -93,8 +94,9 @@ func recordModelLoadFailure(appConfig *config.ApplicationConfig, modelName, back
 func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
 	seen := make(map[string]bool)
 	input := vram.ModelEstimateInput{}
+	managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil

-	addFile := func(uri string) {
+	addFile := func(uri string, size int64) {
 		if !vram.IsWeightFile(uri) {
 			return
 		}
@@ -106,7 +108,7 @@ func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {
 			return
 		}
 		seen[resolved] = true
-		input.Files = append(input.Files, vram.FileInput{URI: resolved})
+		input.Files = append(input.Files, vram.FileInput{URI: resolved, Size: size})
 	}

 	// tryHFRepo resolves any huggingface:// or hf:// URI to an HTTPS URL and
@@ -123,13 +125,25 @@ func estimateModelSizeBytes(c config.ModelConfig, modelsPath string) int64 {

 	for _, f := range c.DownloadFiles {
 		uriStr := string(f.URI)
-		addFile(uriStr)
-		tryHFRepo(uriStr)
+		addFile(uriStr, 0)
+		if !managedPrimary {
+			tryHFRepo(uriStr)
+		}
+	}
+	if managedPrimary {
+		relative := c.ModelFileName()
+		manifest, err := modelartifacts.ReadManifest(filepath.Join(modelsPath, filepath.Dir(relative), "manifest.json"))
+		if err == nil {
+			for _, file := range manifest.Files {
+				addFile(filepath.Join(relative, filepath.FromSlash(file.Path)), file.Size)
+			}
+		}
+	} else {
+		addFile(c.Model, 0)
+		tryHFRepo(c.Model)
 	}
-	addFile(c.Model)
-	tryHFRepo(c.Model)
 	if c.MMProj != "" {
-		addFile(c.MMProj)
+		addFile(c.MMProj, 0)
 	}

 	if len(input.Files) == 0 && input.HFRepo == "" {
@@ -153,6 +167,10 @@ func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...mo
 		model.WithContext(so.Context),
 		model.WithModelID(c.ModelID()),
 	}
+	managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil
+	if managedPrimary {
+		defOpts = append(defOpts, model.WithModelFile(c.ModelFileName()))
+	}

 	threads := 1