feat: materialize Hugging Face model artifacts (#10825)

* feat(config): add model artifact source contract

Assisted-by: Codex:GPT-5 [Codex]

* feat(downloader): add authenticated raw-byte progress

Assisted-by: Codex:GPT-5 [Codex]

* feat(huggingface): resolve immutable snapshot manifests

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): add artifact storage primitives

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): materialize pinned Hugging Face snapshots

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): bind managed snapshots at runtime

Assisted-by: Codex:GPT-5 [Codex]

* feat(gallery): materialize model artifacts during install

Assisted-by: Codex:GPT-5 [Codex]

* feat(gallery): declare managed Hugging Face artifacts

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): preload managed model artifacts

Assisted-by: Codex:GPT-5 [Codex]

* fix(gallery): retain shared artifact caches on delete

Assisted-by: Codex:GPT-5 [Codex]

* feat(models): report artifact acquisition progress

Assisted-by: Codex:GPT-5 [Codex]

* refactor(backends): load managed models from ModelFile

Assisted-by: Codex:GPT-5 [Codex]

* refactor(backends): load staged speech model snapshots

Assisted-by: Codex:GPT-5 [Codex]

* refactor(backends): use staged snapshots in engine backends

Assisted-by: Codex:GPT-5 [Codex]

* test(distributed): cover staged artifact snapshots

Assisted-by: Codex:GPT-5 [Codex]

* docs: explain managed model artifacts

Assisted-by: Codex:GPT-5 [Codex]

* docs: add product design context

Assisted-by: Codex:GPT-5 [Codex]

* feat(ui): show model artifact download progress

Assisted-by: Codex:GPT-5 [Codex]

* Eagerly materialize Hugging Face artifacts

Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback.

Assisted-by: Codex:GPT-5 [shell]

* Refactor HF
  downloads through a shared executor

Assisted-by: Codex:GPT-5 [shell]

* drop

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
2026-07-15 01:09:33 +02:00
committed by GitHub
parent d82c38ee77
commit bcc41219f7
83 changed files with 4315 additions and 339 deletions

View File

@@ -0,0 +1,47 @@
package modelartifacts
import (
"encoding/json"
"fmt"
"os"
)
const ManifestVersion = 1
type Manifest struct {
Version int `json:"version"`
Artifact Spec `json:"artifact"`
Files []ManifestFile `json:"files"`
}
type ManifestFile struct {
Path string `json:"path"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
BlobOID string `json:"blob_oid,omitempty"`
LFSOID string `json:"lfs_oid,omitempty"`
XetHash string `json:"xet_hash,omitempty"`
}
func ReadManifest(fileName string) (Manifest, error) {
data, err := os.ReadFile(fileName)
if err != nil {
return Manifest{}, err
}
var manifest Manifest
if err := json.Unmarshal(data, &manifest); err != nil {
return Manifest{}, err
}
if manifest.Version != ManifestVersion || manifest.Artifact.Resolved == nil {
return Manifest{}, fmt.Errorf("unsupported or incomplete artifact manifest")
}
for _, file := range manifest.Files {
if err := ValidateRelativeHubPath(file.Path); err != nil {
return Manifest{}, err
}
if file.Size < 0 || len(file.SHA256) != 64 {
return Manifest{}, fmt.Errorf("invalid manifest entry %q", file.Path)
}
}
return manifest, nil
}

View File

@@ -0,0 +1,359 @@
package modelartifacts
import (
"context"
"crypto/sha1"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/gofrs/flock"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/pkg/downloader"
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
)
type SnapshotResolver interface {
ResolveSnapshot(context.Context, hfapi.SnapshotRequest) (hfapi.Snapshot, error)
}
type Manager struct {
resolver SnapshotResolver
huggingFaceToken string
}
type ManagerOption func(*Manager)
type Result struct {
Spec Spec
RelativePath string
Manifest Manifest
CacheHit bool
}
func WithHuggingFaceToken(token string) ManagerOption {
return func(manager *Manager) { manager.huggingFaceToken = token }
}
func NewManager(resolver SnapshotResolver, options ...ManagerOption) *Manager {
manager := &Manager{resolver: resolver}
for _, option := range options {
option(manager)
}
return manager
}
func NewDefaultManager(options ...ManagerOption) *Manager {
client := hfapi.NewClient()
client.SetBaseURL(strings.TrimRight(downloader.HF_ENDPOINT, "/") + "/api/models")
return NewManager(client, options...)
}
func committedResult(modelsPath string, spec Spec) (Result, bool) {
if spec.Resolved == nil || spec.Resolved.CacheKey == "" {
return Result{}, false
}
layout, err := LayoutFor(modelsPath, spec)
if err != nil {
return Result{}, false
}
manifest, err := ReadManifest(layout.Manifest)
if err != nil || manifest.Artifact.Resolved == nil || manifest.Artifact.Resolved.CacheKey != spec.Resolved.CacheKey {
return Result{}, false
}
specKey, err := CacheKey(spec)
if err != nil || specKey != spec.Resolved.CacheKey {
return Result{}, false
}
manifestKey, err := CacheKey(manifest.Artifact)
if err != nil || manifestKey != spec.Resolved.CacheKey || len(manifest.Files) == 0 {
return Result{}, false
}
for _, file := range manifest.Files {
info, err := os.Stat(filepath.Join(layout.Snapshot, filepath.FromSlash(file.Path)))
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
return Result{}, false
}
}
relative, err := RelativeSnapshotPath(spec.Resolved.CacheKey)
if err != nil {
return Result{}, false
}
return Result{Spec: spec, RelativePath: relative, Manifest: manifest, CacheHit: true}, true
}
func (m *Manager) Ensure(ctx context.Context, modelsPath string, spec Spec) (Result, error) {
if err := ctx.Err(); err != nil {
return Result{}, err
}
normalized, err := spec.Normalize()
if err != nil {
return Result{}, err
}
if cached, ok := committedResult(modelsPath, normalized); ok {
return cached, nil
}
if m == nil || m.resolver == nil {
return Result{}, fmt.Errorf("artifact materializer has no snapshot resolver")
}
token := ""
if normalized.Source.TokenEnv == HuggingFaceTokenEnv {
token = m.huggingFaceToken
if token == "" {
return Result{}, fmt.Errorf("artifact requires non-empty %s", HuggingFaceTokenEnv)
}
}
revision := normalized.Source.Revision
if normalized.Resolved != nil {
revision = normalized.Resolved.Revision
}
ReportProgress(ctx, ProgressEvent{Phase: PhaseResolving, Artifact: normalized.Name})
snapshot, err := m.resolver.ResolveSnapshot(ctx, hfapi.SnapshotRequest{
Repo: normalized.Source.Repo, Revision: revision, Token: token,
AllowPatterns: normalized.Source.AllowPatterns, IgnorePatterns: normalized.Source.IgnorePatterns,
})
if err != nil {
return Result{}, err
}
if normalized.Resolved != nil && (snapshot.Endpoint != normalized.Resolved.Endpoint || snapshot.ResolvedRevision != normalized.Resolved.Revision) {
return Result{}, fmt.Errorf("resolved artifact identity changed; reinstall the model")
}
normalized.Resolved = &Resolved{Endpoint: snapshot.Endpoint, Revision: snapshot.ResolvedRevision}
cacheKey, err := CacheKey(normalized)
if err != nil {
return Result{}, err
}
normalized.Resolved.CacheKey = cacheKey
layout, err := LayoutFor(modelsPath, normalized)
if err != nil {
return Result{}, err
}
if err := os.MkdirAll(filepath.Dir(layout.Lock), 0o750); err != nil {
return Result{}, err
}
artifactLock := flock.New(layout.Lock)
locked, err := artifactLock.TryLockContext(ctx, 100*time.Millisecond)
if err != nil {
return Result{}, err
}
if !locked {
if err := ctx.Err(); err != nil {
return Result{}, err
}
return Result{}, fmt.Errorf("artifact lock was not acquired")
}
defer func() {
if err := artifactLock.Unlock(); err != nil {
xlog.Warn("failed to unlock model artifact", "lock", layout.Lock, "error", err)
}
}()
if err := os.Chmod(layout.Lock, 0o600); err != nil {
return Result{}, err
}
if cached, ok := committedResult(modelsPath, normalized); ok {
return cached, nil
}
if err := removeInvalidFinal(layout); err != nil {
return Result{}, err
}
return m.materializeLocked(ctx, modelsPath, normalized, snapshot, token, layout)
}
func removeInvalidFinal(layout Layout) error {
root, err := os.OpenRoot(layout.Root)
if err != nil {
return err
}
defer func() { _ = root.Close() }()
relative, err := filepath.Rel(layout.Root, layout.Final)
if err != nil || filepath.Dir(relative) != "huggingface" || !cacheKeyPattern.MatchString(filepath.Base(relative)) {
return fmt.Errorf("refusing to remove invalid artifact path %q", layout.Final)
}
return root.RemoveAll(relative)
}
func (m *Manager) materializeLocked(ctx context.Context, modelsPath string, spec Spec, snapshot hfapi.Snapshot, token string, layout Layout) (Result, error) {
if err := os.MkdirAll(layout.Partial, 0o750); err != nil {
return Result{}, err
}
root, err := os.OpenRoot(layout.Partial)
if err != nil {
return Result{}, err
}
rootClosed := false
defer func() {
if !rootClosed {
_ = root.Close()
}
}()
if err := root.MkdirAll(".downloads", 0o750); err != nil {
return Result{}, err
}
if err := root.MkdirAll("snapshot", 0o750); err != nil {
return Result{}, err
}
totalBytes := int64(0)
if len(snapshot.Files) == 0 {
return Result{}, fmt.Errorf("resolved snapshot contains no selected files")
}
seenPaths := make(map[string]struct{}, len(snapshot.Files))
for _, file := range snapshot.Files {
if err := ValidateRelativeHubPath(file.Path); err != nil {
return Result{}, err
}
if file.Size < 0 || totalBytes > int64(^uint64(0)>>1)-file.Size {
return Result{}, fmt.Errorf("invalid aggregate snapshot size")
}
if _, exists := seenPaths[file.Path]; exists {
return Result{}, fmt.Errorf("duplicate Hub path %q", file.Path)
}
seenPaths[file.Path] = struct{}{}
totalBytes += file.Size
}
manifest := Manifest{Version: ManifestVersion, Artifact: spec, Files: make([]ManifestFile, 0, len(snapshot.Files))}
completedBytes := int64(0)
tasks := make([]downloader.FileTask, 0, len(snapshot.Files))
for index, file := range snapshot.Files {
if err := ctx.Err(); err != nil {
return Result{}, err
}
nameSum := sha256.Sum256([]byte(file.Path))
blobRel := path.Join(".downloads", hex.EncodeToString(nameSum[:]))
blobAbs := filepath.Join(layout.Partial, filepath.FromSlash(blobRel))
file := file
taskIndex := index
task := downloader.FileTask{
URI: downloader.URI(file.URL),
Destination: blobAbs,
SHA256: file.LFSOID,
FileIndex: taskIndex,
TotalFiles: len(snapshot.Files),
Options: []downloader.DownloadOption{
downloader.WithBearerToken(token),
downloader.WithTransferProgress(func(event downloader.TransferProgress) {
ReportProgress(ctx, ProgressEvent{
Phase: PhaseDownloading,
Artifact: spec.Name,
File: file.Path,
CurrentBytes: completedBytes + event.Written,
TotalBytes: totalBytes,
CompletedFiles: taskIndex,
TotalFiles: len(snapshot.Files),
})
}),
},
AfterDownload: func(string) error {
ReportProgress(ctx, ProgressEvent{
Phase: PhaseVerifying,
Artifact: spec.Name,
File: file.Path,
CurrentBytes: completedBytes + file.Size,
TotalBytes: totalBytes,
CompletedFiles: taskIndex,
TotalFiles: len(snapshot.Files),
})
entry, err := verifyDownloadedFile(blobAbs, file)
if err != nil {
_ = root.Remove(blobRel)
return err
}
destination := path.Join("snapshot", file.Path)
if err := root.MkdirAll(path.Dir(destination), 0o750); err != nil {
return err
}
_ = root.Remove(destination)
if err := root.Rename(blobRel, destination); err != nil {
return err
}
manifest.Files = append(manifest.Files, entry)
completedBytes += file.Size
return nil
},
}
tasks = append(tasks, task)
}
if err := downloader.DownloadFilesWithContext(ctx, tasks, nil); err != nil {
return Result{}, err
}
if err := root.RemoveAll(".downloads"); err != nil {
return Result{}, err
}
encoded, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return Result{}, err
}
if err := root.WriteFile("manifest.json.tmp", append(encoded, '\n'), 0o644); err != nil {
return Result{}, err
}
if err := root.Rename("manifest.json.tmp", "manifest.json"); err != nil {
return Result{}, err
}
if err := root.Close(); err != nil {
return Result{}, err
}
rootClosed = true
if err := os.MkdirAll(filepath.Dir(layout.Final), 0o750); err != nil {
return Result{}, err
}
ReportProgress(ctx, ProgressEvent{Phase: PhaseCommitting, Artifact: spec.Name, CurrentBytes: totalBytes, TotalBytes: totalBytes, CompletedFiles: len(snapshot.Files), TotalFiles: len(snapshot.Files)})
if err := os.Rename(layout.Partial, layout.Final); err != nil {
return Result{}, err
}
relative, err := RelativeSnapshotPath(spec.Resolved.CacheKey)
if err != nil {
return Result{}, err
}
return Result{Spec: spec, RelativePath: relative, Manifest: manifest}, nil
}
func verifyDownloadedFile(fileName string, source hfapi.SnapshotFile) (ManifestFile, error) {
file, err := os.Open(fileName)
if err != nil {
return ManifestFile{}, err
}
defer func() { _ = file.Close() }()
sha256Hash := sha256.New()
gitHash := sha1.New()
if _, err := fmt.Fprintf(gitHash, "blob %d%c", source.Size, byte(0)); err != nil {
return ManifestFile{}, err
}
if _, err := io.Copy(io.MultiWriter(sha256Hash, gitHash), file); err != nil {
return ManifestFile{}, err
}
info, err := file.Stat()
if err != nil {
return ManifestFile{}, err
}
if info.Size() != source.Size {
return ManifestFile{}, fmt.Errorf("size mismatch for %q", source.Path)
}
rawSHA256 := hex.EncodeToString(sha256Hash.Sum(nil))
if source.LFSOID != "" {
if decoded, err := hex.DecodeString(source.LFSOID); err != nil || len(decoded) != sha256.Size {
return ManifestFile{}, fmt.Errorf("invalid LFS SHA-256 for %q", source.Path)
}
if !strings.EqualFold(rawSHA256, source.LFSOID) {
return ManifestFile{}, fmt.Errorf("LFS SHA-256 mismatch for %q", source.Path)
}
} else if source.BlobOID != "" {
if decoded, err := hex.DecodeString(source.BlobOID); err != nil || len(decoded) != sha1.Size {
return ManifestFile{}, fmt.Errorf("invalid Git blob OID for %q", source.Path)
}
if !strings.EqualFold(hex.EncodeToString(gitHash.Sum(nil)), source.BlobOID) {
return ManifestFile{}, fmt.Errorf("Git blob OID mismatch for %q", source.Path)
}
}
return ManifestFile{
Path: source.Path, Size: source.Size, SHA256: rawSHA256,
BlobOID: source.BlobOID, LFSOID: source.LFSOID, XetHash: source.XetHash,
}, nil
}

View File

@@ -0,0 +1,153 @@
package modelartifacts_test
import (
"context"
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"sync/atomic"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
type fakeSnapshotResolver struct {
mu sync.Mutex
snapshot hfapi.Snapshot
err error
calls int
}
func (f *fakeSnapshotResolver) ResolveSnapshot(context.Context, hfapi.SnapshotRequest) (hfapi.Snapshot, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls++
return f.snapshot, f.err
}
func (f *fakeSnapshotResolver) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.calls
}
var _ = Describe("controller artifact materializer", func() {
It("requires an injected token before resolving a private artifact", func() {
resolver := &fakeSnapshotResolver{}
_, err := modelartifacts.NewManager(resolver).Ensure(context.Background(), GinkgoT().TempDir(),
modelartifacts.Spec{Source: modelartifacts.Source{
Type: modelartifacts.SourceTypeHuggingFace, Repo: "owner/private", TokenEnv: modelartifacts.HuggingFaceTokenEnv,
}})
Expect(err).To(MatchError(ContainSubstring("non-empty HF_TOKEN")))
Expect(resolver.callCount()).To(BeZero())
})
It("downloads, commits, and reuses a pinned snapshot", func() {
weight := []byte("weight-bytes")
sum := sha256.Sum256(weight)
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
Expect(r.Header.Get("Authorization")).To(Equal("Bearer hf-secret"))
w.Header().Set("Content-Length", "12")
_, _ = w.Write(weight)
}))
DeferCleanup(server.Close)
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
Endpoint: "https://huggingface.co", Repo: "owner/repo",
RequestedRevision: "main", ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
Files: []hfapi.SnapshotFile{{Path: "nested/model.safetensors", Size: 12, LFSOID: hex.EncodeToString(sum[:]), URL: server.URL + "/model"}},
}}
manager := modelartifacts.NewManager(resolver, modelartifacts.WithHuggingFaceToken("hf-secret"))
modelsPath := GinkgoT().TempDir()
spec := modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", TokenEnv: "HF_TOKEN"}}
result, err := manager.Ensure(context.Background(), modelsPath, spec)
Expect(err).NotTo(HaveOccurred())
Expect(result.CacheHit).To(BeFalse())
Expect(result.Spec.Resolved.Revision).To(Equal("0123456789abcdef0123456789abcdef01234567"))
Expect(result.RelativePath).To(HavePrefix(".artifacts/huggingface/"))
Expect(os.ReadFile(filepath.Join(modelsPath, filepath.FromSlash(result.RelativePath), "nested", "model.safetensors"))).To(Equal(weight))
manifestBytes, err := os.ReadFile(filepath.Join(modelsPath, filepath.Dir(filepath.FromSlash(result.RelativePath)), "manifest.json"))
Expect(err).NotTo(HaveOccurred())
Expect(string(manifestBytes)).NotTo(ContainSubstring("hf-secret"))
cached, err := manager.Ensure(context.Background(), modelsPath, result.Spec)
Expect(err).NotTo(HaveOccurred())
Expect(cached.CacheHit).To(BeTrue())
Expect(requests.Load()).To(Equal(int32(1)))
Expect(resolver.callCount()).To(Equal(1))
})
It("rejects a path escape before opening a destination", func() {
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
Endpoint: "https://huggingface.co", Repo: "owner/repo",
ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
Files: []hfapi.SnapshotFile{{Path: "../escape", Size: 1, URL: "https://example.invalid/file"}},
}}
modelsPath := GinkgoT().TempDir()
_, err := modelartifacts.NewManager(resolver).Ensure(context.Background(), modelsPath,
modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}})
Expect(err).To(MatchError(ContainSubstring("unsafe Hub path")))
Expect(filepath.Join(modelsPath, "escape")).NotTo(BeAnExistingFile())
})
It("does not commit after cancellation", func() {
started := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(started)
<-r.Context().Done()
}))
DeferCleanup(server.Close)
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
Endpoint: "https://huggingface.co", Repo: "owner/repo",
ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
Files: []hfapi.SnapshotFile{{Path: "model.bin", Size: 10, URL: server.URL}},
}}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
modelsPath := GinkgoT().TempDir()
go func() {
_, err := modelartifacts.NewManager(resolver).Ensure(ctx, modelsPath,
modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}})
done <- err
}()
<-started
cancel()
Expect(<-done).To(MatchError(context.Canceled))
entries, err := filepath.Glob(filepath.Join(modelsPath, ".artifacts", "huggingface", "*"))
Expect(err).NotTo(HaveOccurred())
Expect(entries).To(BeEmpty())
})
It("emits resolving, downloading, verifying, and committing phases", func() {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("x")) }))
DeferCleanup(server.Close)
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
Endpoint: "https://huggingface.co", Repo: "owner/repo",
ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
Files: []hfapi.SnapshotFile{{Path: "config.json", Size: 1, URL: server.URL}},
}}
var phases []modelartifacts.Phase
ctx := modelartifacts.WithProgressSink(context.Background(), func(event modelartifacts.ProgressEvent) {
if len(phases) == 0 || phases[len(phases)-1] != event.Phase {
phases = append(phases, event.Phase)
}
})
_, err := modelartifacts.NewManager(resolver).Ensure(ctx, GinkgoT().TempDir(),
modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}})
Expect(err).NotTo(HaveOccurred())
Expect(phases).To(Equal([]modelartifacts.Phase{
modelartifacts.PhaseResolving, modelartifacts.PhaseDownloading, modelartifacts.PhaseVerifying, modelartifacts.PhaseCommitting,
}))
})
})

View File

@@ -0,0 +1,13 @@
package modelartifacts_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestModelArtifacts(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Model Artifacts Suite")
}

View File

@@ -0,0 +1,86 @@
package modelartifacts
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"path/filepath"
"strings"
)
type Layout struct {
Root string
Lock string
Partial string
PartialSnapshot string
Final string
Snapshot string
Manifest string
}
func CacheKey(spec Spec) (string, error) {
normalized, err := spec.Normalize()
if err != nil {
return "", err
}
if normalized.Resolved == nil {
return "", fmt.Errorf("cache key requires resolved artifact state")
}
identity := struct {
Type string `json:"type"`
Endpoint string `json:"endpoint"`
Repo string `json:"repo"`
Revision string `json:"revision"`
AllowPatterns []string `json:"allow_patterns,omitempty"`
IgnorePatterns []string `json:"ignore_patterns,omitempty"`
}{
Type: normalized.Source.Type, Endpoint: normalized.Resolved.Endpoint,
Repo: normalized.Source.Repo, Revision: normalized.Resolved.Revision,
AllowPatterns: normalized.Source.AllowPatterns, IgnorePatterns: normalized.Source.IgnorePatterns,
}
encoded, err := json.Marshal(identity)
if err != nil {
return "", err
}
sum := sha256.Sum256(encoded)
return hex.EncodeToString(sum[:]), nil
}
func RelativeSnapshotPath(cacheKey string) (string, error) {
if !cacheKeyPattern.MatchString(cacheKey) {
return "", fmt.Errorf("invalid artifact cache key")
}
return filepath.Join(".artifacts", "huggingface", cacheKey, "snapshot"), nil
}
func LayoutFor(modelsPath string, spec Spec) (Layout, error) {
if spec.Resolved == nil || !cacheKeyPattern.MatchString(spec.Resolved.CacheKey) {
return Layout{}, fmt.Errorf("layout requires a resolved cache key")
}
root := filepath.Join(modelsPath, ".artifacts")
partial := filepath.Join(root, ".partial", spec.Resolved.CacheKey)
final := filepath.Join(root, "huggingface", spec.Resolved.CacheKey)
return Layout{
Root: root,
Lock: filepath.Join(root, ".locks", spec.Resolved.CacheKey+".lock"),
Partial: partial,
PartialSnapshot: filepath.Join(partial, "snapshot"),
Final: final,
Snapshot: filepath.Join(final, "snapshot"),
Manifest: filepath.Join(final, "manifest.json"),
}, nil
}
func ValidateRelativeHubPath(candidate string) error {
if candidate == "" || filepath.IsAbs(candidate) || strings.ContainsAny(candidate, "\\\x00") {
return fmt.Errorf("unsafe Hub path %q", candidate)
}
parts := strings.Split(candidate, "/")
for _, part := range parts {
if part == "" || part == "." || part == ".." {
return fmt.Errorf("unsafe Hub path %q", candidate)
}
}
return nil
}

View File

@@ -0,0 +1,109 @@
package modelartifacts_test
import (
"context"
"encoding/json"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
var _ = Describe("artifact storage primitives", func() {
baseSpec := func() modelartifacts.Spec {
return modelartifacts.Spec{
Name: "model",
Target: "model",
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
},
}
}
It("creates a stable path-safe cache identity", func() {
first, err := modelartifacts.CacheKey(baseSpec())
Expect(err).NotTo(HaveOccurred())
second := baseSpec()
second.Source.AllowPatterns = []string{"*.json", "*.safetensors"}
third := second
third.Source.AllowPatterns = []string{"*.safetensors", "*.json"}
secondKey, err := modelartifacts.CacheKey(second)
Expect(err).NotTo(HaveOccurred())
thirdKey, err := modelartifacts.CacheKey(third)
Expect(err).NotTo(HaveOccurred())
Expect(first).To(MatchRegexp(`^[0-9a-f]{64}$`))
Expect(secondKey).To(Equal(thirdKey))
Expect(secondKey).NotTo(Equal(first))
})
It("places all state below the models artifact root", func() {
spec := baseSpec()
key, err := modelartifacts.CacheKey(spec)
Expect(err).NotTo(HaveOccurred())
spec.Resolved.CacheKey = key
layout, err := modelartifacts.LayoutFor("/models", spec)
Expect(err).NotTo(HaveOccurred())
Expect(layout.Final).To(Equal(filepath.Join("/models", ".artifacts", "huggingface", key)))
Expect(layout.Snapshot).To(Equal(filepath.Join(layout.Final, "snapshot")))
relative, err := modelartifacts.RelativeSnapshotPath(key)
Expect(err).NotTo(HaveOccurred())
Expect(relative).To(Equal(filepath.Join(".artifacts", "huggingface", key, "snapshot")))
_, err = modelartifacts.RelativeSnapshotPath("sha256:" + key)
Expect(err).To(MatchError(ContainSubstring("invalid artifact cache key")))
})
DescribeTable("rejects hostile Hub paths",
func(candidate string) { Expect(modelartifacts.ValidateRelativeHubPath(candidate)).NotTo(Succeed()) },
Entry("absolute", "/etc/passwd"),
Entry("parent", "nested/../escape"),
Entry("backslash", `nested\escape`),
Entry("NUL", "bad\x00path"),
Entry("empty component", "nested//file"),
)
It("reads a complete versioned manifest", func() {
spec := baseSpec()
key, err := modelartifacts.CacheKey(spec)
Expect(err).NotTo(HaveOccurred())
spec.Resolved.CacheKey = key
encoded, err := json.Marshal(modelartifacts.Manifest{
Version: modelartifacts.ManifestVersion,
Artifact: spec,
Files: []modelartifacts.ManifestFile{{
Path: "nested/model.safetensors",
Size: 11,
SHA256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
}},
})
Expect(err).NotTo(HaveOccurred())
fileName := filepath.Join(GinkgoT().TempDir(), "manifest.json")
Expect(os.WriteFile(fileName, encoded, 0o600)).To(Succeed())
manifest, err := modelartifacts.ReadManifest(fileName)
Expect(err).NotTo(HaveOccurred())
Expect(manifest.Artifact.Resolved.CacheKey).To(Equal(key))
Expect(manifest.Files).To(HaveLen(1))
})
It("reports progress only through the request-scoped sink", func() {
event := modelartifacts.ProgressEvent{
Phase: modelartifacts.PhaseDownloading,
Artifact: "model",
File: "weights.safetensors",
CurrentBytes: 7,
TotalBytes: 11,
}
var received []modelartifacts.ProgressEvent
ctx := modelartifacts.WithProgressSink(context.Background(), func(update modelartifacts.ProgressEvent) {
received = append(received, update)
})
modelartifacts.ReportProgress(ctx, event)
modelartifacts.ReportProgress(context.Background(), event)
Expect(received).To(Equal([]modelartifacts.ProgressEvent{event}))
})
})

View File

@@ -0,0 +1,40 @@
package modelartifacts
import "context"
type Phase string
const (
PhaseResolving Phase = "resolving"
PhaseDownloading Phase = "downloading"
PhaseVerifying Phase = "verifying"
PhaseCommitting Phase = "committing"
PhasePersisting Phase = "persisting"
)
type ProgressEvent struct {
Phase Phase
Artifact string
File string
CurrentBytes int64
TotalBytes int64
CompletedFiles int
TotalFiles int
}
type ProgressSink func(ProgressEvent)
type progressSinkKey struct{}
func WithProgressSink(ctx context.Context, sink ProgressSink) context.Context {
if sink == nil {
return ctx
}
return context.WithValue(ctx, progressSinkKey{}, sink)
}
func ReportProgress(ctx context.Context, event ProgressEvent) {
if sink, ok := ctx.Value(progressSinkKey{}).(ProgressSink); ok && sink != nil {
sink(event)
}
}

View File

@@ -0,0 +1,82 @@
package modelartifacts
import (
"fmt"
"strings"
)
var hfReferencePrefixes = []string{
"https://huggingface.co/",
"huggingface://",
"hf://",
"hf.co/",
}
// ParsePrimaryReference converts a Hugging Face repository or file reference
// into a managed artifact spec. It accepts repo roots like "owner/repo" and
// direct file references like "huggingface://owner/repo/path/to/model.gguf".
// The boolean return is false when the reference is not Hugging Face-shaped.
func ParsePrimaryReference(raw string) (Spec, bool, error) {
source, ok, err := ParsePrimarySource(raw)
if err != nil || !ok {
return Spec{}, ok, err
}
return Spec{
Name: TargetModel,
Target: TargetModel,
Source: source,
}, true, nil
}
// ParsePrimarySource converts a Hugging Face repository or file reference into
// a managed source definition. Direct file references are translated into a
// repo plus a single allow pattern so the materializer downloads only the
// selected file.
func ParsePrimarySource(raw string) (Source, bool, error) {
ref := strings.TrimSpace(raw)
if ref == "" {
return Source{}, false, nil
}
stripped := ref
for _, prefix := range hfReferencePrefixes {
stripped = strings.TrimPrefix(stripped, prefix)
}
stripped = strings.TrimSuffix(stripped, "/")
parts := strings.Split(stripped, "/")
if len(parts) < 2 {
return Source{}, false, nil
}
hadPrefix := ref != stripped
repo, err := normalizeRepo(strings.Join(parts[:2], "/"))
if err != nil {
if ref != stripped {
return Source{}, false, fmt.Errorf("invalid Hugging Face reference %q: %w", raw, err)
}
return Source{}, false, nil
}
if !hadPrefix && len(parts) == 2 {
return Source{}, false, nil
}
source := Source{
Type: SourceTypeHuggingFace,
Repo: repo,
}
if len(parts) == 2 {
return source, true, nil
}
if parts[2] == "resolve" {
if len(parts) < 5 {
return Source{}, false, fmt.Errorf("invalid Hugging Face file reference %q", raw)
}
source.AllowPatterns = []string{strings.Join(parts[4:], "/")}
return source, true, nil
}
source.AllowPatterns = []string{strings.Join(parts[2:], "/")}
return source, true, nil
}

160
pkg/modelartifacts/types.go Normal file
View File

@@ -0,0 +1,160 @@
package modelartifacts
import (
"fmt"
"net/url"
"path"
"regexp"
"slices"
"strings"
)
const (
SourceTypeHuggingFace = "huggingface"
TargetModel = "model"
HuggingFaceTokenEnv = "HF_TOKEN"
)
var (
commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
cacheKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
)
type Spec struct {
Name string `yaml:"name" json:"name"`
Target string `yaml:"target" json:"target"`
Source Source `yaml:"source" json:"source"`
Resolved *Resolved `yaml:"resolved,omitempty" json:"resolved,omitempty"`
}
type Source struct {
Type string `yaml:"type" json:"type"`
Repo string `yaml:"repo" json:"repo"`
Revision string `yaml:"revision,omitempty" json:"revision,omitempty"`
TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"`
AllowPatterns []string `yaml:"allow_patterns,omitempty" json:"allow_patterns,omitempty"`
IgnorePatterns []string `yaml:"ignore_patterns,omitempty" json:"ignore_patterns,omitempty"`
}
type Resolved struct {
Endpoint string `yaml:"endpoint" json:"endpoint"`
Revision string `yaml:"revision" json:"revision"`
CacheKey string `yaml:"cache_key" json:"cache_key"`
}
func (s Spec) Normalize() (Spec, error) {
if strings.TrimSpace(s.Name) == "" {
s.Name = TargetModel
}
if strings.TrimSpace(s.Target) == "" {
s.Target = TargetModel
}
s.Name = strings.TrimSpace(s.Name)
s.Target = strings.TrimSpace(s.Target)
s.Source.Type = strings.TrimSpace(s.Source.Type)
s.Source.TokenEnv = strings.TrimSpace(s.Source.TokenEnv)
if s.Name != TargetModel || s.Target != TargetModel {
return Spec{}, fmt.Errorf("only artifact name/target %q is supported", TargetModel)
}
if s.Source.Type != SourceTypeHuggingFace {
return Spec{}, fmt.Errorf("unsupported artifact source type %q", s.Source.Type)
}
repo, err := normalizeRepo(s.Source.Repo)
if err != nil {
return Spec{}, err
}
s.Source.Repo = repo
if strings.TrimSpace(s.Source.Revision) == "" {
s.Source.Revision = "main"
} else {
s.Source.Revision = strings.TrimSpace(s.Source.Revision)
}
if strings.ContainsRune(s.Source.Revision, '\x00') {
return Spec{}, fmt.Errorf("revision contains NUL")
}
if s.Source.TokenEnv != "" && s.Source.TokenEnv != HuggingFaceTokenEnv {
return Spec{}, fmt.Errorf("token_env must be empty or %s", HuggingFaceTokenEnv)
}
for _, patterns := range [][]string{s.Source.AllowPatterns, s.Source.IgnorePatterns} {
for _, pattern := range patterns {
if err := validatePattern(pattern); err != nil {
return Spec{}, err
}
}
}
s.Source.AllowPatterns = slices.Clone(s.Source.AllowPatterns)
s.Source.IgnorePatterns = slices.Clone(s.Source.IgnorePatterns)
slices.Sort(s.Source.AllowPatterns)
slices.Sort(s.Source.IgnorePatterns)
if s.Resolved != nil {
resolved := *s.Resolved
resolved.Endpoint, err = normalizeEndpoint(resolved.Endpoint)
if err != nil {
return Spec{}, err
}
resolved.Revision = strings.ToLower(strings.TrimSpace(resolved.Revision))
if !commitPattern.MatchString(resolved.Revision) {
return Spec{}, fmt.Errorf("resolved revision must be 40 lowercase hexadecimal characters")
}
if resolved.CacheKey != "" && !cacheKeyPattern.MatchString(resolved.CacheKey) {
return Spec{}, fmt.Errorf("resolved cache key must be 64 lowercase hexadecimal characters")
}
s.Resolved = &resolved
}
return s, nil
}
func (s Spec) Validate() error {
normalized, err := s.Normalize()
if err != nil {
return err
}
if normalized.Resolved != nil && normalized.Resolved.CacheKey == "" {
return fmt.Errorf("resolved cache key is required in installed state")
}
return nil
}
func normalizeRepo(raw string) (string, error) {
repo := strings.TrimSpace(raw)
for _, prefix := range []string{"https://huggingface.co/", "huggingface://", "hf://"} {
repo = strings.TrimPrefix(repo, prefix)
}
repo = strings.TrimSuffix(repo, "/")
parts := strings.Split(repo, "/")
if len(parts) != 2 || parts[0] == "" || parts[1] == "" || strings.ContainsAny(repo, "?#\\\x00") {
return "", fmt.Errorf("Hugging Face repo must be exactly owner/repo")
}
return repo, nil
}
// CanonicalRepo normalizes a Hugging Face repository reference to owner/repo.
func CanonicalRepo(raw string) (string, error) {
return normalizeRepo(raw)
}
func validatePattern(pattern string) error {
if pattern == "" || path.IsAbs(pattern) || strings.ContainsAny(pattern, "\\\x00") {
return fmt.Errorf("invalid artifact pattern %q", pattern)
}
for _, part := range strings.Split(pattern, "/") {
if part == ".." {
return fmt.Errorf("invalid artifact pattern %q", pattern)
}
}
return nil
}
func normalizeEndpoint(raw string) (string, error) {
u, err := url.Parse(strings.TrimSpace(raw))
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return "", fmt.Errorf("invalid resolved Hugging Face endpoint")
}
u.Path = strings.TrimRight(u.Path, "/")
return u.String(), nil
}

View File

@@ -0,0 +1,70 @@
package modelartifacts_test
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/pkg/modelartifacts"
)
var _ = Describe("artifact configuration", func() {
It("normalizes the supported primary Hugging Face source", func() {
spec, err := (modelartifacts.Spec{
Source: modelartifacts.Source{
Type: "huggingface",
Repo: "hf://Qwen/Qwen3-ASR-1.7B",
TokenEnv: "HF_TOKEN",
},
}).Normalize()
Expect(err).NotTo(HaveOccurred())
Expect(spec.Name).To(Equal("model"))
Expect(spec.Target).To(Equal("model"))
Expect(spec.Source.Repo).To(Equal("Qwen/Qwen3-ASR-1.7B"))
Expect(spec.Source.Revision).To(Equal("main"))
})
It("parses Hugging Face repo and file references into managed sources", func() {
repo, ok, err := modelartifacts.ParsePrimarySource("huggingface://Qwen/Qwen3-ASR-1.7B")
Expect(err).NotTo(HaveOccurred())
Expect(ok).To(BeTrue())
Expect(repo.Type).To(Equal(modelartifacts.SourceTypeHuggingFace))
Expect(repo.Repo).To(Equal("Qwen/Qwen3-ASR-1.7B"))
Expect(repo.AllowPatterns).To(BeEmpty())
file, ok, err := modelartifacts.ParsePrimarySource("https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf")
Expect(err).NotTo(HaveOccurred())
Expect(ok).To(BeTrue())
Expect(file.Repo).To(Equal("nomic-ai/nomic-embed-text-v1.5-GGUF"))
Expect(file.AllowPatterns).To(Equal([]string{"nomic-embed-text-v1.5.f16.gguf"}))
})
It("ignores non-Hugging Face references", func() {
_, ok, err := modelartifacts.ParsePrimarySource("models/local-model.gguf")
Expect(err).NotTo(HaveOccurred())
Expect(ok).To(BeFalse())
})
DescribeTable("rejects unsafe or unsupported declarations",
func(spec modelartifacts.Spec, message string) {
Expect(spec.Validate()).To(MatchError(ContainSubstring(message)))
},
Entry("unknown source", modelartifacts.Spec{Source: modelartifacts.Source{Type: "s3", Repo: "owner/repo"}}, "source type"),
Entry("secondary target", modelartifacts.Spec{Name: "controlnet", Target: "controlnet", Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}}, "target"),
Entry("malformed repo", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo/file"}}, "owner/repo"),
Entry("unrelated secret", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", TokenEnv: "AWS_SECRET_ACCESS_KEY"}}, "HF_TOKEN"),
Entry("parent filter", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", AllowPatterns: []string{"../*.json"}}}, "pattern"),
Entry("prefixed cache key", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}, Resolved: &modelartifacts.Resolved{Endpoint: "https://huggingface.co", Revision: "0123456789abcdef0123456789abcdef01234567", CacheKey: "sha256:bad"}}, "cache key"),
)
It("validates installed state", func() {
spec := modelartifacts.Spec{
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
Resolved: &modelartifacts.Resolved{
Endpoint: "https://huggingface.co",
Revision: "0123456789abcdef0123456789abcdef01234567",
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
},
}
Expect(spec.Validate()).To(Succeed())
})
})