feat: materialize Hugging Face model artifacts (#10825)

* feat(config): add model artifact source contract Assisted-by: Codex:GPT-5 [Codex] * feat(downloader): add authenticated raw-byte progress Assisted-by: Codex:GPT-5 [Codex] * feat(huggingface): resolve immutable snapshot manifests Assisted-by: Codex:GPT-5 [Codex] * feat(models): add artifact storage primitives Assisted-by: Codex:GPT-5 [Codex] * feat(models): materialize pinned Hugging Face snapshots Assisted-by: Codex:GPT-5 [Codex] * feat(models): bind managed snapshots at runtime Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): materialize model artifacts during install Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): declare managed Hugging Face artifacts Assisted-by: Codex:GPT-5 [Codex] * feat(models): preload managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * fix(gallery): retain shared artifact caches on delete Assisted-by: Codex:GPT-5 [Codex] * feat(models): report artifact acquisition progress Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load managed models from ModelFile Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load staged speech model snapshots Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): use staged snapshots in engine backends Assisted-by: Codex:GPT-5 [Codex] * test(distributed): cover staged artifact snapshots Assisted-by: Codex:GPT-5 [Codex] * docs: explain managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * docs: add product design context Assisted-by: Codex:GPT-5 [Codex] * feat(ui): show model artifact download progress Assisted-by: Codex:GPT-5 [Codex] * Eagerly materialize Hugging Face artifacts Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback. Assisted-by: Codex:GPT-5 [shell] * Refactor HF downloads through a shared executor Assisted-by: Codex:GPT-5 [shell] * drop Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
2026-07-31 10:28:43 -04:00 · 2026-07-15 01:09:33 +02:00
parent d82c38ee77
commit bcc41219f7
83 changed files with 4315 additions and 339 deletions
--- a/pkg/modelartifacts/path.go
+++ b/pkg/modelartifacts/path.go
@@ -0,0 +1,86 @@
+package modelartifacts
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type Layout struct {
+	Root            string
+	Lock            string
+	Partial         string
+	PartialSnapshot string
+	Final           string
+	Snapshot        string
+	Manifest        string
+}
+
+func CacheKey(spec Spec) (string, error) {
+	normalized, err := spec.Normalize()
+	if err != nil {
+		return "", err
+	}
+	if normalized.Resolved == nil {
+		return "", fmt.Errorf("cache key requires resolved artifact state")
+	}
+	identity := struct {
+		Type           string   `json:"type"`
+		Endpoint       string   `json:"endpoint"`
+		Repo           string   `json:"repo"`
+		Revision       string   `json:"revision"`
+		AllowPatterns  []string `json:"allow_patterns,omitempty"`
+		IgnorePatterns []string `json:"ignore_patterns,omitempty"`
+	}{
+		Type: normalized.Source.Type, Endpoint: normalized.Resolved.Endpoint,
+		Repo: normalized.Source.Repo, Revision: normalized.Resolved.Revision,
+		AllowPatterns: normalized.Source.AllowPatterns, IgnorePatterns: normalized.Source.IgnorePatterns,
+	}
+	encoded, err := json.Marshal(identity)
+	if err != nil {
+		return "", err
+	}
+	sum := sha256.Sum256(encoded)
+	return hex.EncodeToString(sum[:]), nil
+}
+
+func RelativeSnapshotPath(cacheKey string) (string, error) {
+	if !cacheKeyPattern.MatchString(cacheKey) {
+		return "", fmt.Errorf("invalid artifact cache key")
+	}
+	return filepath.Join(".artifacts", "huggingface", cacheKey, "snapshot"), nil
+}
+
+func LayoutFor(modelsPath string, spec Spec) (Layout, error) {
+	if spec.Resolved == nil || !cacheKeyPattern.MatchString(spec.Resolved.CacheKey) {
+		return Layout{}, fmt.Errorf("layout requires a resolved cache key")
+	}
+	root := filepath.Join(modelsPath, ".artifacts")
+	partial := filepath.Join(root, ".partial", spec.Resolved.CacheKey)
+	final := filepath.Join(root, "huggingface", spec.Resolved.CacheKey)
+	return Layout{
+		Root:            root,
+		Lock:            filepath.Join(root, ".locks", spec.Resolved.CacheKey+".lock"),
+		Partial:         partial,
+		PartialSnapshot: filepath.Join(partial, "snapshot"),
+		Final:           final,
+		Snapshot:        filepath.Join(final, "snapshot"),
+		Manifest:        filepath.Join(final, "manifest.json"),
+	}, nil
+}
+
+func ValidateRelativeHubPath(candidate string) error {
+	if candidate == "" || filepath.IsAbs(candidate) || strings.ContainsAny(candidate, "\\\x00") {
+		return fmt.Errorf("unsafe Hub path %q", candidate)
+	}
+	parts := strings.Split(candidate, "/")
+	for _, part := range parts {
+		if part == "" || part == "." || part == ".." {
+			return fmt.Errorf("unsafe Hub path %q", candidate)
+		}
+	}
+	return nil
+}