mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
feat: materialize Hugging Face model artifacts (#10825)
* feat(config): add model artifact source contract Assisted-by: Codex:GPT-5 [Codex] * feat(downloader): add authenticated raw-byte progress Assisted-by: Codex:GPT-5 [Codex] * feat(huggingface): resolve immutable snapshot manifests Assisted-by: Codex:GPT-5 [Codex] * feat(models): add artifact storage primitives Assisted-by: Codex:GPT-5 [Codex] * feat(models): materialize pinned Hugging Face snapshots Assisted-by: Codex:GPT-5 [Codex] * feat(models): bind managed snapshots at runtime Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): materialize model artifacts during install Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): declare managed Hugging Face artifacts Assisted-by: Codex:GPT-5 [Codex] * feat(models): preload managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * fix(gallery): retain shared artifact caches on delete Assisted-by: Codex:GPT-5 [Codex] * feat(models): report artifact acquisition progress Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load managed models from ModelFile Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load staged speech model snapshots Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): use staged snapshots in engine backends Assisted-by: Codex:GPT-5 [Codex] * test(distributed): cover staged artifact snapshots Assisted-by: Codex:GPT-5 [Codex] * docs: explain managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * docs: add product design context Assisted-by: Codex:GPT-5 [Codex] * feat(ui): show model artifact download progress Assisted-by: Codex:GPT-5 [Codex] * Eagerly materialize Hugging Face artifacts Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback. Assisted-by: Codex:GPT-5 [shell] * Refactor HF downloads through a shared executor Assisted-by: Codex:GPT-5 [shell] * drop Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
47
pkg/modelartifacts/manifest.go
Normal file
47
pkg/modelartifacts/manifest.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package modelartifacts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
const ManifestVersion = 1
|
||||
|
||||
type Manifest struct {
|
||||
Version int `json:"version"`
|
||||
Artifact Spec `json:"artifact"`
|
||||
Files []ManifestFile `json:"files"`
|
||||
}
|
||||
|
||||
type ManifestFile struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
BlobOID string `json:"blob_oid,omitempty"`
|
||||
LFSOID string `json:"lfs_oid,omitempty"`
|
||||
XetHash string `json:"xet_hash,omitempty"`
|
||||
}
|
||||
|
||||
func ReadManifest(fileName string) (Manifest, error) {
|
||||
data, err := os.ReadFile(fileName)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
var manifest Manifest
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
if manifest.Version != ManifestVersion || manifest.Artifact.Resolved == nil {
|
||||
return Manifest{}, fmt.Errorf("unsupported or incomplete artifact manifest")
|
||||
}
|
||||
for _, file := range manifest.Files {
|
||||
if err := ValidateRelativeHubPath(file.Path); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
if file.Size < 0 || len(file.SHA256) != 64 {
|
||||
return Manifest{}, fmt.Errorf("invalid manifest entry %q", file.Path)
|
||||
}
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
359
pkg/modelartifacts/materializer.go
Normal file
359
pkg/modelartifacts/materializer.go
Normal file
@@ -0,0 +1,359 @@
|
||||
package modelartifacts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofrs/flock"
|
||||
"github.com/mudler/xlog"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
|
||||
)
|
||||
|
||||
type SnapshotResolver interface {
|
||||
ResolveSnapshot(context.Context, hfapi.SnapshotRequest) (hfapi.Snapshot, error)
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
resolver SnapshotResolver
|
||||
huggingFaceToken string
|
||||
}
|
||||
|
||||
type ManagerOption func(*Manager)
|
||||
|
||||
type Result struct {
|
||||
Spec Spec
|
||||
RelativePath string
|
||||
Manifest Manifest
|
||||
CacheHit bool
|
||||
}
|
||||
|
||||
func WithHuggingFaceToken(token string) ManagerOption {
|
||||
return func(manager *Manager) { manager.huggingFaceToken = token }
|
||||
}
|
||||
|
||||
func NewManager(resolver SnapshotResolver, options ...ManagerOption) *Manager {
|
||||
manager := &Manager{resolver: resolver}
|
||||
for _, option := range options {
|
||||
option(manager)
|
||||
}
|
||||
return manager
|
||||
}
|
||||
|
||||
func NewDefaultManager(options ...ManagerOption) *Manager {
|
||||
client := hfapi.NewClient()
|
||||
client.SetBaseURL(strings.TrimRight(downloader.HF_ENDPOINT, "/") + "/api/models")
|
||||
return NewManager(client, options...)
|
||||
}
|
||||
|
||||
func committedResult(modelsPath string, spec Spec) (Result, bool) {
|
||||
if spec.Resolved == nil || spec.Resolved.CacheKey == "" {
|
||||
return Result{}, false
|
||||
}
|
||||
layout, err := LayoutFor(modelsPath, spec)
|
||||
if err != nil {
|
||||
return Result{}, false
|
||||
}
|
||||
manifest, err := ReadManifest(layout.Manifest)
|
||||
if err != nil || manifest.Artifact.Resolved == nil || manifest.Artifact.Resolved.CacheKey != spec.Resolved.CacheKey {
|
||||
return Result{}, false
|
||||
}
|
||||
specKey, err := CacheKey(spec)
|
||||
if err != nil || specKey != spec.Resolved.CacheKey {
|
||||
return Result{}, false
|
||||
}
|
||||
manifestKey, err := CacheKey(manifest.Artifact)
|
||||
if err != nil || manifestKey != spec.Resolved.CacheKey || len(manifest.Files) == 0 {
|
||||
return Result{}, false
|
||||
}
|
||||
for _, file := range manifest.Files {
|
||||
info, err := os.Stat(filepath.Join(layout.Snapshot, filepath.FromSlash(file.Path)))
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
|
||||
return Result{}, false
|
||||
}
|
||||
}
|
||||
relative, err := RelativeSnapshotPath(spec.Resolved.CacheKey)
|
||||
if err != nil {
|
||||
return Result{}, false
|
||||
}
|
||||
return Result{Spec: spec, RelativePath: relative, Manifest: manifest, CacheHit: true}, true
|
||||
}
|
||||
|
||||
func (m *Manager) Ensure(ctx context.Context, modelsPath string, spec Spec) (Result, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
normalized, err := spec.Normalize()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if cached, ok := committedResult(modelsPath, normalized); ok {
|
||||
return cached, nil
|
||||
}
|
||||
if m == nil || m.resolver == nil {
|
||||
return Result{}, fmt.Errorf("artifact materializer has no snapshot resolver")
|
||||
}
|
||||
token := ""
|
||||
if normalized.Source.TokenEnv == HuggingFaceTokenEnv {
|
||||
token = m.huggingFaceToken
|
||||
if token == "" {
|
||||
return Result{}, fmt.Errorf("artifact requires non-empty %s", HuggingFaceTokenEnv)
|
||||
}
|
||||
}
|
||||
revision := normalized.Source.Revision
|
||||
if normalized.Resolved != nil {
|
||||
revision = normalized.Resolved.Revision
|
||||
}
|
||||
ReportProgress(ctx, ProgressEvent{Phase: PhaseResolving, Artifact: normalized.Name})
|
||||
snapshot, err := m.resolver.ResolveSnapshot(ctx, hfapi.SnapshotRequest{
|
||||
Repo: normalized.Source.Repo, Revision: revision, Token: token,
|
||||
AllowPatterns: normalized.Source.AllowPatterns, IgnorePatterns: normalized.Source.IgnorePatterns,
|
||||
})
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if normalized.Resolved != nil && (snapshot.Endpoint != normalized.Resolved.Endpoint || snapshot.ResolvedRevision != normalized.Resolved.Revision) {
|
||||
return Result{}, fmt.Errorf("resolved artifact identity changed; reinstall the model")
|
||||
}
|
||||
normalized.Resolved = &Resolved{Endpoint: snapshot.Endpoint, Revision: snapshot.ResolvedRevision}
|
||||
cacheKey, err := CacheKey(normalized)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
normalized.Resolved.CacheKey = cacheKey
|
||||
layout, err := LayoutFor(modelsPath, normalized)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(layout.Lock), 0o750); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
artifactLock := flock.New(layout.Lock)
|
||||
locked, err := artifactLock.TryLockContext(ctx, 100*time.Millisecond)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if !locked {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{}, fmt.Errorf("artifact lock was not acquired")
|
||||
}
|
||||
defer func() {
|
||||
if err := artifactLock.Unlock(); err != nil {
|
||||
xlog.Warn("failed to unlock model artifact", "lock", layout.Lock, "error", err)
|
||||
}
|
||||
}()
|
||||
if err := os.Chmod(layout.Lock, 0o600); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if cached, ok := committedResult(modelsPath, normalized); ok {
|
||||
return cached, nil
|
||||
}
|
||||
if err := removeInvalidFinal(layout); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return m.materializeLocked(ctx, modelsPath, normalized, snapshot, token, layout)
|
||||
}
|
||||
|
||||
func removeInvalidFinal(layout Layout) error {
|
||||
root, err := os.OpenRoot(layout.Root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
relative, err := filepath.Rel(layout.Root, layout.Final)
|
||||
if err != nil || filepath.Dir(relative) != "huggingface" || !cacheKeyPattern.MatchString(filepath.Base(relative)) {
|
||||
return fmt.Errorf("refusing to remove invalid artifact path %q", layout.Final)
|
||||
}
|
||||
return root.RemoveAll(relative)
|
||||
}
|
||||
|
||||
func (m *Manager) materializeLocked(ctx context.Context, modelsPath string, spec Spec, snapshot hfapi.Snapshot, token string, layout Layout) (Result, error) {
|
||||
if err := os.MkdirAll(layout.Partial, 0o750); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
root, err := os.OpenRoot(layout.Partial)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
rootClosed := false
|
||||
defer func() {
|
||||
if !rootClosed {
|
||||
_ = root.Close()
|
||||
}
|
||||
}()
|
||||
if err := root.MkdirAll(".downloads", 0o750); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := root.MkdirAll("snapshot", 0o750); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
totalBytes := int64(0)
|
||||
if len(snapshot.Files) == 0 {
|
||||
return Result{}, fmt.Errorf("resolved snapshot contains no selected files")
|
||||
}
|
||||
seenPaths := make(map[string]struct{}, len(snapshot.Files))
|
||||
for _, file := range snapshot.Files {
|
||||
if err := ValidateRelativeHubPath(file.Path); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if file.Size < 0 || totalBytes > int64(^uint64(0)>>1)-file.Size {
|
||||
return Result{}, fmt.Errorf("invalid aggregate snapshot size")
|
||||
}
|
||||
if _, exists := seenPaths[file.Path]; exists {
|
||||
return Result{}, fmt.Errorf("duplicate Hub path %q", file.Path)
|
||||
}
|
||||
seenPaths[file.Path] = struct{}{}
|
||||
totalBytes += file.Size
|
||||
}
|
||||
|
||||
manifest := Manifest{Version: ManifestVersion, Artifact: spec, Files: make([]ManifestFile, 0, len(snapshot.Files))}
|
||||
completedBytes := int64(0)
|
||||
tasks := make([]downloader.FileTask, 0, len(snapshot.Files))
|
||||
for index, file := range snapshot.Files {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
nameSum := sha256.Sum256([]byte(file.Path))
|
||||
blobRel := path.Join(".downloads", hex.EncodeToString(nameSum[:]))
|
||||
blobAbs := filepath.Join(layout.Partial, filepath.FromSlash(blobRel))
|
||||
file := file
|
||||
taskIndex := index
|
||||
task := downloader.FileTask{
|
||||
URI: downloader.URI(file.URL),
|
||||
Destination: blobAbs,
|
||||
SHA256: file.LFSOID,
|
||||
FileIndex: taskIndex,
|
||||
TotalFiles: len(snapshot.Files),
|
||||
Options: []downloader.DownloadOption{
|
||||
downloader.WithBearerToken(token),
|
||||
downloader.WithTransferProgress(func(event downloader.TransferProgress) {
|
||||
ReportProgress(ctx, ProgressEvent{
|
||||
Phase: PhaseDownloading,
|
||||
Artifact: spec.Name,
|
||||
File: file.Path,
|
||||
CurrentBytes: completedBytes + event.Written,
|
||||
TotalBytes: totalBytes,
|
||||
CompletedFiles: taskIndex,
|
||||
TotalFiles: len(snapshot.Files),
|
||||
})
|
||||
}),
|
||||
},
|
||||
AfterDownload: func(string) error {
|
||||
ReportProgress(ctx, ProgressEvent{
|
||||
Phase: PhaseVerifying,
|
||||
Artifact: spec.Name,
|
||||
File: file.Path,
|
||||
CurrentBytes: completedBytes + file.Size,
|
||||
TotalBytes: totalBytes,
|
||||
CompletedFiles: taskIndex,
|
||||
TotalFiles: len(snapshot.Files),
|
||||
})
|
||||
entry, err := verifyDownloadedFile(blobAbs, file)
|
||||
if err != nil {
|
||||
_ = root.Remove(blobRel)
|
||||
return err
|
||||
}
|
||||
destination := path.Join("snapshot", file.Path)
|
||||
if err := root.MkdirAll(path.Dir(destination), 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = root.Remove(destination)
|
||||
if err := root.Rename(blobRel, destination); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest.Files = append(manifest.Files, entry)
|
||||
completedBytes += file.Size
|
||||
return nil
|
||||
},
|
||||
}
|
||||
tasks = append(tasks, task)
|
||||
}
|
||||
if err := downloader.DownloadFilesWithContext(ctx, tasks, nil); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := root.RemoveAll(".downloads"); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
encoded, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := root.WriteFile("manifest.json.tmp", append(encoded, '\n'), 0o644); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := root.Rename("manifest.json.tmp", "manifest.json"); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := root.Close(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
rootClosed = true
|
||||
if err := os.MkdirAll(filepath.Dir(layout.Final), 0o750); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
ReportProgress(ctx, ProgressEvent{Phase: PhaseCommitting, Artifact: spec.Name, CurrentBytes: totalBytes, TotalBytes: totalBytes, CompletedFiles: len(snapshot.Files), TotalFiles: len(snapshot.Files)})
|
||||
if err := os.Rename(layout.Partial, layout.Final); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
relative, err := RelativeSnapshotPath(spec.Resolved.CacheKey)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{Spec: spec, RelativePath: relative, Manifest: manifest}, nil
|
||||
}
|
||||
|
||||
func verifyDownloadedFile(fileName string, source hfapi.SnapshotFile) (ManifestFile, error) {
|
||||
file, err := os.Open(fileName)
|
||||
if err != nil {
|
||||
return ManifestFile{}, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
sha256Hash := sha256.New()
|
||||
gitHash := sha1.New()
|
||||
if _, err := fmt.Fprintf(gitHash, "blob %d%c", source.Size, byte(0)); err != nil {
|
||||
return ManifestFile{}, err
|
||||
}
|
||||
if _, err := io.Copy(io.MultiWriter(sha256Hash, gitHash), file); err != nil {
|
||||
return ManifestFile{}, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return ManifestFile{}, err
|
||||
}
|
||||
if info.Size() != source.Size {
|
||||
return ManifestFile{}, fmt.Errorf("size mismatch for %q", source.Path)
|
||||
}
|
||||
rawSHA256 := hex.EncodeToString(sha256Hash.Sum(nil))
|
||||
if source.LFSOID != "" {
|
||||
if decoded, err := hex.DecodeString(source.LFSOID); err != nil || len(decoded) != sha256.Size {
|
||||
return ManifestFile{}, fmt.Errorf("invalid LFS SHA-256 for %q", source.Path)
|
||||
}
|
||||
if !strings.EqualFold(rawSHA256, source.LFSOID) {
|
||||
return ManifestFile{}, fmt.Errorf("LFS SHA-256 mismatch for %q", source.Path)
|
||||
}
|
||||
} else if source.BlobOID != "" {
|
||||
if decoded, err := hex.DecodeString(source.BlobOID); err != nil || len(decoded) != sha1.Size {
|
||||
return ManifestFile{}, fmt.Errorf("invalid Git blob OID for %q", source.Path)
|
||||
}
|
||||
if !strings.EqualFold(hex.EncodeToString(gitHash.Sum(nil)), source.BlobOID) {
|
||||
return ManifestFile{}, fmt.Errorf("Git blob OID mismatch for %q", source.Path)
|
||||
}
|
||||
}
|
||||
return ManifestFile{
|
||||
Path: source.Path, Size: source.Size, SHA256: rawSHA256,
|
||||
BlobOID: source.BlobOID, LFSOID: source.LFSOID, XetHash: source.XetHash,
|
||||
}, nil
|
||||
}
|
||||
153
pkg/modelartifacts/materializer_test.go
Normal file
153
pkg/modelartifacts/materializer_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package modelartifacts_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
type fakeSnapshotResolver struct {
|
||||
mu sync.Mutex
|
||||
snapshot hfapi.Snapshot
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeSnapshotResolver) ResolveSnapshot(context.Context, hfapi.SnapshotRequest) (hfapi.Snapshot, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls++
|
||||
return f.snapshot, f.err
|
||||
}
|
||||
|
||||
func (f *fakeSnapshotResolver) callCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.calls
|
||||
}
|
||||
|
||||
var _ = Describe("controller artifact materializer", func() {
|
||||
It("requires an injected token before resolving a private artifact", func() {
|
||||
resolver := &fakeSnapshotResolver{}
|
||||
_, err := modelartifacts.NewManager(resolver).Ensure(context.Background(), GinkgoT().TempDir(),
|
||||
modelartifacts.Spec{Source: modelartifacts.Source{
|
||||
Type: modelartifacts.SourceTypeHuggingFace, Repo: "owner/private", TokenEnv: modelartifacts.HuggingFaceTokenEnv,
|
||||
}})
|
||||
Expect(err).To(MatchError(ContainSubstring("non-empty HF_TOKEN")))
|
||||
Expect(resolver.callCount()).To(BeZero())
|
||||
})
|
||||
|
||||
It("downloads, commits, and reuses a pinned snapshot", func() {
|
||||
weight := []byte("weight-bytes")
|
||||
sum := sha256.Sum256(weight)
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
Expect(r.Header.Get("Authorization")).To(Equal("Bearer hf-secret"))
|
||||
w.Header().Set("Content-Length", "12")
|
||||
_, _ = w.Write(weight)
|
||||
}))
|
||||
DeferCleanup(server.Close)
|
||||
|
||||
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
|
||||
Endpoint: "https://huggingface.co", Repo: "owner/repo",
|
||||
RequestedRevision: "main", ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
|
||||
Files: []hfapi.SnapshotFile{{Path: "nested/model.safetensors", Size: 12, LFSOID: hex.EncodeToString(sum[:]), URL: server.URL + "/model"}},
|
||||
}}
|
||||
manager := modelartifacts.NewManager(resolver, modelartifacts.WithHuggingFaceToken("hf-secret"))
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
spec := modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", TokenEnv: "HF_TOKEN"}}
|
||||
|
||||
result, err := manager.Ensure(context.Background(), modelsPath, spec)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result.CacheHit).To(BeFalse())
|
||||
Expect(result.Spec.Resolved.Revision).To(Equal("0123456789abcdef0123456789abcdef01234567"))
|
||||
Expect(result.RelativePath).To(HavePrefix(".artifacts/huggingface/"))
|
||||
Expect(os.ReadFile(filepath.Join(modelsPath, filepath.FromSlash(result.RelativePath), "nested", "model.safetensors"))).To(Equal(weight))
|
||||
|
||||
manifestBytes, err := os.ReadFile(filepath.Join(modelsPath, filepath.Dir(filepath.FromSlash(result.RelativePath)), "manifest.json"))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(manifestBytes)).NotTo(ContainSubstring("hf-secret"))
|
||||
|
||||
cached, err := manager.Ensure(context.Background(), modelsPath, result.Spec)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(cached.CacheHit).To(BeTrue())
|
||||
Expect(requests.Load()).To(Equal(int32(1)))
|
||||
Expect(resolver.callCount()).To(Equal(1))
|
||||
})
|
||||
|
||||
It("rejects a path escape before opening a destination", func() {
|
||||
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
|
||||
Endpoint: "https://huggingface.co", Repo: "owner/repo",
|
||||
ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
|
||||
Files: []hfapi.SnapshotFile{{Path: "../escape", Size: 1, URL: "https://example.invalid/file"}},
|
||||
}}
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
_, err := modelartifacts.NewManager(resolver).Ensure(context.Background(), modelsPath,
|
||||
modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}})
|
||||
Expect(err).To(MatchError(ContainSubstring("unsafe Hub path")))
|
||||
Expect(filepath.Join(modelsPath, "escape")).NotTo(BeAnExistingFile())
|
||||
})
|
||||
|
||||
It("does not commit after cancellation", func() {
|
||||
started := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(started)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
DeferCleanup(server.Close)
|
||||
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
|
||||
Endpoint: "https://huggingface.co", Repo: "owner/repo",
|
||||
ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
|
||||
Files: []hfapi.SnapshotFile{{Path: "model.bin", Size: 10, URL: server.URL}},
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
modelsPath := GinkgoT().TempDir()
|
||||
go func() {
|
||||
_, err := modelartifacts.NewManager(resolver).Ensure(ctx, modelsPath,
|
||||
modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}})
|
||||
done <- err
|
||||
}()
|
||||
<-started
|
||||
cancel()
|
||||
Expect(<-done).To(MatchError(context.Canceled))
|
||||
entries, err := filepath.Glob(filepath.Join(modelsPath, ".artifacts", "huggingface", "*"))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(entries).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("emits resolving, downloading, verifying, and committing phases", func() {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("x")) }))
|
||||
DeferCleanup(server.Close)
|
||||
resolver := &fakeSnapshotResolver{snapshot: hfapi.Snapshot{
|
||||
Endpoint: "https://huggingface.co", Repo: "owner/repo",
|
||||
ResolvedRevision: "0123456789abcdef0123456789abcdef01234567",
|
||||
Files: []hfapi.SnapshotFile{{Path: "config.json", Size: 1, URL: server.URL}},
|
||||
}}
|
||||
var phases []modelartifacts.Phase
|
||||
ctx := modelartifacts.WithProgressSink(context.Background(), func(event modelartifacts.ProgressEvent) {
|
||||
if len(phases) == 0 || phases[len(phases)-1] != event.Phase {
|
||||
phases = append(phases, event.Phase)
|
||||
}
|
||||
})
|
||||
_, err := modelartifacts.NewManager(resolver).Ensure(ctx, GinkgoT().TempDir(),
|
||||
modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(phases).To(Equal([]modelartifacts.Phase{
|
||||
modelartifacts.PhaseResolving, modelartifacts.PhaseDownloading, modelartifacts.PhaseVerifying, modelartifacts.PhaseCommitting,
|
||||
}))
|
||||
})
|
||||
})
|
||||
13
pkg/modelartifacts/modelartifacts_suite_test.go
Normal file
13
pkg/modelartifacts/modelartifacts_suite_test.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package modelartifacts_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestModelArtifacts(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Model Artifacts Suite")
|
||||
}
|
||||
86
pkg/modelartifacts/path.go
Normal file
86
pkg/modelartifacts/path.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package modelartifacts
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Layout struct {
|
||||
Root string
|
||||
Lock string
|
||||
Partial string
|
||||
PartialSnapshot string
|
||||
Final string
|
||||
Snapshot string
|
||||
Manifest string
|
||||
}
|
||||
|
||||
func CacheKey(spec Spec) (string, error) {
|
||||
normalized, err := spec.Normalize()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if normalized.Resolved == nil {
|
||||
return "", fmt.Errorf("cache key requires resolved artifact state")
|
||||
}
|
||||
identity := struct {
|
||||
Type string `json:"type"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Repo string `json:"repo"`
|
||||
Revision string `json:"revision"`
|
||||
AllowPatterns []string `json:"allow_patterns,omitempty"`
|
||||
IgnorePatterns []string `json:"ignore_patterns,omitempty"`
|
||||
}{
|
||||
Type: normalized.Source.Type, Endpoint: normalized.Resolved.Endpoint,
|
||||
Repo: normalized.Source.Repo, Revision: normalized.Resolved.Revision,
|
||||
AllowPatterns: normalized.Source.AllowPatterns, IgnorePatterns: normalized.Source.IgnorePatterns,
|
||||
}
|
||||
encoded, err := json.Marshal(identity)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(encoded)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func RelativeSnapshotPath(cacheKey string) (string, error) {
|
||||
if !cacheKeyPattern.MatchString(cacheKey) {
|
||||
return "", fmt.Errorf("invalid artifact cache key")
|
||||
}
|
||||
return filepath.Join(".artifacts", "huggingface", cacheKey, "snapshot"), nil
|
||||
}
|
||||
|
||||
func LayoutFor(modelsPath string, spec Spec) (Layout, error) {
|
||||
if spec.Resolved == nil || !cacheKeyPattern.MatchString(spec.Resolved.CacheKey) {
|
||||
return Layout{}, fmt.Errorf("layout requires a resolved cache key")
|
||||
}
|
||||
root := filepath.Join(modelsPath, ".artifacts")
|
||||
partial := filepath.Join(root, ".partial", spec.Resolved.CacheKey)
|
||||
final := filepath.Join(root, "huggingface", spec.Resolved.CacheKey)
|
||||
return Layout{
|
||||
Root: root,
|
||||
Lock: filepath.Join(root, ".locks", spec.Resolved.CacheKey+".lock"),
|
||||
Partial: partial,
|
||||
PartialSnapshot: filepath.Join(partial, "snapshot"),
|
||||
Final: final,
|
||||
Snapshot: filepath.Join(final, "snapshot"),
|
||||
Manifest: filepath.Join(final, "manifest.json"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ValidateRelativeHubPath(candidate string) error {
|
||||
if candidate == "" || filepath.IsAbs(candidate) || strings.ContainsAny(candidate, "\\\x00") {
|
||||
return fmt.Errorf("unsafe Hub path %q", candidate)
|
||||
}
|
||||
parts := strings.Split(candidate, "/")
|
||||
for _, part := range parts {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return fmt.Errorf("unsafe Hub path %q", candidate)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
109
pkg/modelartifacts/path_test.go
Normal file
109
pkg/modelartifacts/path_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package modelartifacts_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
var _ = Describe("artifact storage primitives", func() {
|
||||
baseSpec := func() modelartifacts.Spec {
|
||||
return modelartifacts.Spec{
|
||||
Name: "model",
|
||||
Target: "model",
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
It("creates a stable path-safe cache identity", func() {
|
||||
first, err := modelartifacts.CacheKey(baseSpec())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
second := baseSpec()
|
||||
second.Source.AllowPatterns = []string{"*.json", "*.safetensors"}
|
||||
third := second
|
||||
third.Source.AllowPatterns = []string{"*.safetensors", "*.json"}
|
||||
secondKey, err := modelartifacts.CacheKey(second)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
thirdKey, err := modelartifacts.CacheKey(third)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(first).To(MatchRegexp(`^[0-9a-f]{64}$`))
|
||||
Expect(secondKey).To(Equal(thirdKey))
|
||||
Expect(secondKey).NotTo(Equal(first))
|
||||
})
|
||||
|
||||
It("places all state below the models artifact root", func() {
|
||||
spec := baseSpec()
|
||||
key, err := modelartifacts.CacheKey(spec)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
spec.Resolved.CacheKey = key
|
||||
layout, err := modelartifacts.LayoutFor("/models", spec)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(layout.Final).To(Equal(filepath.Join("/models", ".artifacts", "huggingface", key)))
|
||||
Expect(layout.Snapshot).To(Equal(filepath.Join(layout.Final, "snapshot")))
|
||||
relative, err := modelartifacts.RelativeSnapshotPath(key)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(relative).To(Equal(filepath.Join(".artifacts", "huggingface", key, "snapshot")))
|
||||
_, err = modelartifacts.RelativeSnapshotPath("sha256:" + key)
|
||||
Expect(err).To(MatchError(ContainSubstring("invalid artifact cache key")))
|
||||
})
|
||||
|
||||
DescribeTable("rejects hostile Hub paths",
|
||||
func(candidate string) { Expect(modelartifacts.ValidateRelativeHubPath(candidate)).NotTo(Succeed()) },
|
||||
Entry("absolute", "/etc/passwd"),
|
||||
Entry("parent", "nested/../escape"),
|
||||
Entry("backslash", `nested\escape`),
|
||||
Entry("NUL", "bad\x00path"),
|
||||
Entry("empty component", "nested//file"),
|
||||
)
|
||||
|
||||
It("reads a complete versioned manifest", func() {
|
||||
spec := baseSpec()
|
||||
key, err := modelartifacts.CacheKey(spec)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
spec.Resolved.CacheKey = key
|
||||
encoded, err := json.Marshal(modelartifacts.Manifest{
|
||||
Version: modelartifacts.ManifestVersion,
|
||||
Artifact: spec,
|
||||
Files: []modelartifacts.ManifestFile{{
|
||||
Path: "nested/model.safetensors",
|
||||
Size: 11,
|
||||
SHA256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
}},
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
fileName := filepath.Join(GinkgoT().TempDir(), "manifest.json")
|
||||
Expect(os.WriteFile(fileName, encoded, 0o600)).To(Succeed())
|
||||
|
||||
manifest, err := modelartifacts.ReadManifest(fileName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(manifest.Artifact.Resolved.CacheKey).To(Equal(key))
|
||||
Expect(manifest.Files).To(HaveLen(1))
|
||||
})
|
||||
|
||||
It("reports progress only through the request-scoped sink", func() {
|
||||
event := modelartifacts.ProgressEvent{
|
||||
Phase: modelartifacts.PhaseDownloading,
|
||||
Artifact: "model",
|
||||
File: "weights.safetensors",
|
||||
CurrentBytes: 7,
|
||||
TotalBytes: 11,
|
||||
}
|
||||
var received []modelartifacts.ProgressEvent
|
||||
ctx := modelartifacts.WithProgressSink(context.Background(), func(update modelartifacts.ProgressEvent) {
|
||||
received = append(received, update)
|
||||
})
|
||||
modelartifacts.ReportProgress(ctx, event)
|
||||
modelartifacts.ReportProgress(context.Background(), event)
|
||||
Expect(received).To(Equal([]modelartifacts.ProgressEvent{event}))
|
||||
})
|
||||
})
|
||||
40
pkg/modelartifacts/progress.go
Normal file
40
pkg/modelartifacts/progress.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package modelartifacts
|
||||
|
||||
import "context"
|
||||
|
||||
type Phase string
|
||||
|
||||
const (
|
||||
PhaseResolving Phase = "resolving"
|
||||
PhaseDownloading Phase = "downloading"
|
||||
PhaseVerifying Phase = "verifying"
|
||||
PhaseCommitting Phase = "committing"
|
||||
PhasePersisting Phase = "persisting"
|
||||
)
|
||||
|
||||
type ProgressEvent struct {
|
||||
Phase Phase
|
||||
Artifact string
|
||||
File string
|
||||
CurrentBytes int64
|
||||
TotalBytes int64
|
||||
CompletedFiles int
|
||||
TotalFiles int
|
||||
}
|
||||
|
||||
type ProgressSink func(ProgressEvent)
|
||||
|
||||
type progressSinkKey struct{}
|
||||
|
||||
func WithProgressSink(ctx context.Context, sink ProgressSink) context.Context {
|
||||
if sink == nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, progressSinkKey{}, sink)
|
||||
}
|
||||
|
||||
func ReportProgress(ctx context.Context, event ProgressEvent) {
|
||||
if sink, ok := ctx.Value(progressSinkKey{}).(ProgressSink); ok && sink != nil {
|
||||
sink(event)
|
||||
}
|
||||
}
|
||||
82
pkg/modelartifacts/reference.go
Normal file
82
pkg/modelartifacts/reference.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package modelartifacts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var hfReferencePrefixes = []string{
|
||||
"https://huggingface.co/",
|
||||
"huggingface://",
|
||||
"hf://",
|
||||
"hf.co/",
|
||||
}
|
||||
|
||||
// ParsePrimaryReference converts a Hugging Face repository or file reference
|
||||
// into a managed artifact spec. It accepts repo roots like "owner/repo" and
|
||||
// direct file references like "huggingface://owner/repo/path/to/model.gguf".
|
||||
// The boolean return is false when the reference is not Hugging Face-shaped.
|
||||
func ParsePrimaryReference(raw string) (Spec, bool, error) {
|
||||
source, ok, err := ParsePrimarySource(raw)
|
||||
if err != nil || !ok {
|
||||
return Spec{}, ok, err
|
||||
}
|
||||
return Spec{
|
||||
Name: TargetModel,
|
||||
Target: TargetModel,
|
||||
Source: source,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
// ParsePrimarySource converts a Hugging Face repository or file reference into
|
||||
// a managed source definition. Direct file references are translated into a
|
||||
// repo plus a single allow pattern so the materializer downloads only the
|
||||
// selected file.
|
||||
func ParsePrimarySource(raw string) (Source, bool, error) {
|
||||
ref := strings.TrimSpace(raw)
|
||||
if ref == "" {
|
||||
return Source{}, false, nil
|
||||
}
|
||||
|
||||
stripped := ref
|
||||
for _, prefix := range hfReferencePrefixes {
|
||||
stripped = strings.TrimPrefix(stripped, prefix)
|
||||
}
|
||||
stripped = strings.TrimSuffix(stripped, "/")
|
||||
|
||||
parts := strings.Split(stripped, "/")
|
||||
if len(parts) < 2 {
|
||||
return Source{}, false, nil
|
||||
}
|
||||
hadPrefix := ref != stripped
|
||||
|
||||
repo, err := normalizeRepo(strings.Join(parts[:2], "/"))
|
||||
if err != nil {
|
||||
if ref != stripped {
|
||||
return Source{}, false, fmt.Errorf("invalid Hugging Face reference %q: %w", raw, err)
|
||||
}
|
||||
return Source{}, false, nil
|
||||
}
|
||||
if !hadPrefix && len(parts) == 2 {
|
||||
return Source{}, false, nil
|
||||
}
|
||||
|
||||
source := Source{
|
||||
Type: SourceTypeHuggingFace,
|
||||
Repo: repo,
|
||||
}
|
||||
if len(parts) == 2 {
|
||||
return source, true, nil
|
||||
}
|
||||
|
||||
if parts[2] == "resolve" {
|
||||
if len(parts) < 5 {
|
||||
return Source{}, false, fmt.Errorf("invalid Hugging Face file reference %q", raw)
|
||||
}
|
||||
source.AllowPatterns = []string{strings.Join(parts[4:], "/")}
|
||||
return source, true, nil
|
||||
}
|
||||
|
||||
source.AllowPatterns = []string{strings.Join(parts[2:], "/")}
|
||||
return source, true, nil
|
||||
}
|
||||
160
pkg/modelartifacts/types.go
Normal file
160
pkg/modelartifacts/types.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package modelartifacts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceTypeHuggingFace = "huggingface"
|
||||
TargetModel = "model"
|
||||
HuggingFaceTokenEnv = "HF_TOKEN"
|
||||
)
|
||||
|
||||
var (
|
||||
commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
cacheKeyPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
)
|
||||
|
||||
type Spec struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Target string `yaml:"target" json:"target"`
|
||||
Source Source `yaml:"source" json:"source"`
|
||||
Resolved *Resolved `yaml:"resolved,omitempty" json:"resolved,omitempty"`
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
Type string `yaml:"type" json:"type"`
|
||||
Repo string `yaml:"repo" json:"repo"`
|
||||
Revision string `yaml:"revision,omitempty" json:"revision,omitempty"`
|
||||
TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"`
|
||||
AllowPatterns []string `yaml:"allow_patterns,omitempty" json:"allow_patterns,omitempty"`
|
||||
IgnorePatterns []string `yaml:"ignore_patterns,omitempty" json:"ignore_patterns,omitempty"`
|
||||
}
|
||||
|
||||
type Resolved struct {
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Revision string `yaml:"revision" json:"revision"`
|
||||
CacheKey string `yaml:"cache_key" json:"cache_key"`
|
||||
}
|
||||
|
||||
func (s Spec) Normalize() (Spec, error) {
|
||||
if strings.TrimSpace(s.Name) == "" {
|
||||
s.Name = TargetModel
|
||||
}
|
||||
if strings.TrimSpace(s.Target) == "" {
|
||||
s.Target = TargetModel
|
||||
}
|
||||
s.Name = strings.TrimSpace(s.Name)
|
||||
s.Target = strings.TrimSpace(s.Target)
|
||||
s.Source.Type = strings.TrimSpace(s.Source.Type)
|
||||
s.Source.TokenEnv = strings.TrimSpace(s.Source.TokenEnv)
|
||||
|
||||
if s.Name != TargetModel || s.Target != TargetModel {
|
||||
return Spec{}, fmt.Errorf("only artifact name/target %q is supported", TargetModel)
|
||||
}
|
||||
if s.Source.Type != SourceTypeHuggingFace {
|
||||
return Spec{}, fmt.Errorf("unsupported artifact source type %q", s.Source.Type)
|
||||
}
|
||||
|
||||
repo, err := normalizeRepo(s.Source.Repo)
|
||||
if err != nil {
|
||||
return Spec{}, err
|
||||
}
|
||||
s.Source.Repo = repo
|
||||
if strings.TrimSpace(s.Source.Revision) == "" {
|
||||
s.Source.Revision = "main"
|
||||
} else {
|
||||
s.Source.Revision = strings.TrimSpace(s.Source.Revision)
|
||||
}
|
||||
if strings.ContainsRune(s.Source.Revision, '\x00') {
|
||||
return Spec{}, fmt.Errorf("revision contains NUL")
|
||||
}
|
||||
if s.Source.TokenEnv != "" && s.Source.TokenEnv != HuggingFaceTokenEnv {
|
||||
return Spec{}, fmt.Errorf("token_env must be empty or %s", HuggingFaceTokenEnv)
|
||||
}
|
||||
|
||||
for _, patterns := range [][]string{s.Source.AllowPatterns, s.Source.IgnorePatterns} {
|
||||
for _, pattern := range patterns {
|
||||
if err := validatePattern(pattern); err != nil {
|
||||
return Spec{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
s.Source.AllowPatterns = slices.Clone(s.Source.AllowPatterns)
|
||||
s.Source.IgnorePatterns = slices.Clone(s.Source.IgnorePatterns)
|
||||
slices.Sort(s.Source.AllowPatterns)
|
||||
slices.Sort(s.Source.IgnorePatterns)
|
||||
|
||||
if s.Resolved != nil {
|
||||
resolved := *s.Resolved
|
||||
resolved.Endpoint, err = normalizeEndpoint(resolved.Endpoint)
|
||||
if err != nil {
|
||||
return Spec{}, err
|
||||
}
|
||||
resolved.Revision = strings.ToLower(strings.TrimSpace(resolved.Revision))
|
||||
if !commitPattern.MatchString(resolved.Revision) {
|
||||
return Spec{}, fmt.Errorf("resolved revision must be 40 lowercase hexadecimal characters")
|
||||
}
|
||||
if resolved.CacheKey != "" && !cacheKeyPattern.MatchString(resolved.CacheKey) {
|
||||
return Spec{}, fmt.Errorf("resolved cache key must be 64 lowercase hexadecimal characters")
|
||||
}
|
||||
s.Resolved = &resolved
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s Spec) Validate() error {
|
||||
normalized, err := s.Normalize()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if normalized.Resolved != nil && normalized.Resolved.CacheKey == "" {
|
||||
return fmt.Errorf("resolved cache key is required in installed state")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeRepo(raw string) (string, error) {
|
||||
repo := strings.TrimSpace(raw)
|
||||
for _, prefix := range []string{"https://huggingface.co/", "huggingface://", "hf://"} {
|
||||
repo = strings.TrimPrefix(repo, prefix)
|
||||
}
|
||||
repo = strings.TrimSuffix(repo, "/")
|
||||
parts := strings.Split(repo, "/")
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" || strings.ContainsAny(repo, "?#\\\x00") {
|
||||
return "", fmt.Errorf("Hugging Face repo must be exactly owner/repo")
|
||||
}
|
||||
return repo, nil
|
||||
}
|
||||
|
||||
// CanonicalRepo normalizes a Hugging Face repository reference to owner/repo.
|
||||
func CanonicalRepo(raw string) (string, error) {
|
||||
return normalizeRepo(raw)
|
||||
}
|
||||
|
||||
func validatePattern(pattern string) error {
|
||||
if pattern == "" || path.IsAbs(pattern) || strings.ContainsAny(pattern, "\\\x00") {
|
||||
return fmt.Errorf("invalid artifact pattern %q", pattern)
|
||||
}
|
||||
for _, part := range strings.Split(pattern, "/") {
|
||||
if part == ".." {
|
||||
return fmt.Errorf("invalid artifact pattern %q", pattern)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeEndpoint(raw string) (string, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
|
||||
return "", fmt.Errorf("invalid resolved Hugging Face endpoint")
|
||||
}
|
||||
u.Path = strings.TrimRight(u.Path, "/")
|
||||
return u.String(), nil
|
||||
}
|
||||
70
pkg/modelartifacts/types_test.go
Normal file
70
pkg/modelartifacts/types_test.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package modelartifacts_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
||||
)
|
||||
|
||||
var _ = Describe("artifact configuration", func() {
|
||||
It("normalizes the supported primary Hugging Face source", func() {
|
||||
spec, err := (modelartifacts.Spec{
|
||||
Source: modelartifacts.Source{
|
||||
Type: "huggingface",
|
||||
Repo: "hf://Qwen/Qwen3-ASR-1.7B",
|
||||
TokenEnv: "HF_TOKEN",
|
||||
},
|
||||
}).Normalize()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(spec.Name).To(Equal("model"))
|
||||
Expect(spec.Target).To(Equal("model"))
|
||||
Expect(spec.Source.Repo).To(Equal("Qwen/Qwen3-ASR-1.7B"))
|
||||
Expect(spec.Source.Revision).To(Equal("main"))
|
||||
})
|
||||
|
||||
It("parses Hugging Face repo and file references into managed sources", func() {
|
||||
repo, ok, err := modelartifacts.ParsePrimarySource("huggingface://Qwen/Qwen3-ASR-1.7B")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(repo.Type).To(Equal(modelartifacts.SourceTypeHuggingFace))
|
||||
Expect(repo.Repo).To(Equal("Qwen/Qwen3-ASR-1.7B"))
|
||||
Expect(repo.AllowPatterns).To(BeEmpty())
|
||||
|
||||
file, ok, err := modelartifacts.ParsePrimarySource("https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(file.Repo).To(Equal("nomic-ai/nomic-embed-text-v1.5-GGUF"))
|
||||
Expect(file.AllowPatterns).To(Equal([]string{"nomic-embed-text-v1.5.f16.gguf"}))
|
||||
})
|
||||
|
||||
It("ignores non-Hugging Face references", func() {
|
||||
_, ok, err := modelartifacts.ParsePrimarySource("models/local-model.gguf")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(ok).To(BeFalse())
|
||||
})
|
||||
|
||||
DescribeTable("rejects unsafe or unsupported declarations",
|
||||
func(spec modelartifacts.Spec, message string) {
|
||||
Expect(spec.Validate()).To(MatchError(ContainSubstring(message)))
|
||||
},
|
||||
Entry("unknown source", modelartifacts.Spec{Source: modelartifacts.Source{Type: "s3", Repo: "owner/repo"}}, "source type"),
|
||||
Entry("secondary target", modelartifacts.Spec{Name: "controlnet", Target: "controlnet", Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}}, "target"),
|
||||
Entry("malformed repo", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo/file"}}, "owner/repo"),
|
||||
Entry("unrelated secret", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", TokenEnv: "AWS_SECRET_ACCESS_KEY"}}, "HF_TOKEN"),
|
||||
Entry("parent filter", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", AllowPatterns: []string{"../*.json"}}}, "pattern"),
|
||||
Entry("prefixed cache key", modelartifacts.Spec{Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"}, Resolved: &modelartifacts.Resolved{Endpoint: "https://huggingface.co", Revision: "0123456789abcdef0123456789abcdef01234567", CacheKey: "sha256:bad"}}, "cache key"),
|
||||
)
|
||||
|
||||
It("validates installed state", func() {
|
||||
spec := modelartifacts.Spec{
|
||||
Source: modelartifacts.Source{Type: "huggingface", Repo: "owner/repo"},
|
||||
Resolved: &modelartifacts.Resolved{
|
||||
Endpoint: "https://huggingface.co",
|
||||
Revision: "0123456789abcdef0123456789abcdef01234567",
|
||||
CacheKey: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
},
|
||||
}
|
||||
Expect(spec.Validate()).To(Succeed())
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user