mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 18:09:05 -04:00
feat: materialize Hugging Face model artifacts (#10825)
* feat(config): add model artifact source contract Assisted-by: Codex:GPT-5 [Codex] * feat(downloader): add authenticated raw-byte progress Assisted-by: Codex:GPT-5 [Codex] * feat(huggingface): resolve immutable snapshot manifests Assisted-by: Codex:GPT-5 [Codex] * feat(models): add artifact storage primitives Assisted-by: Codex:GPT-5 [Codex] * feat(models): materialize pinned Hugging Face snapshots Assisted-by: Codex:GPT-5 [Codex] * feat(models): bind managed snapshots at runtime Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): materialize model artifacts during install Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): declare managed Hugging Face artifacts Assisted-by: Codex:GPT-5 [Codex] * feat(models): preload managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * fix(gallery): retain shared artifact caches on delete Assisted-by: Codex:GPT-5 [Codex] * feat(models): report artifact acquisition progress Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load managed models from ModelFile Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load staged speech model snapshots Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): use staged snapshots in engine backends Assisted-by: Codex:GPT-5 [Codex] * test(distributed): cover staged artifact snapshots Assisted-by: Codex:GPT-5 [Codex] * docs: explain managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * docs: add product design context Assisted-by: Codex:GPT-5 [Codex] * feat(ui): show model artifact download progress Assisted-by: Codex:GPT-5 [Codex] * Eagerly materialize Hugging Face artifacts Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback. Assisted-by: Codex:GPT-5 [shell] * Refactor HF downloads through a shared executor Assisted-by: Codex:GPT-5 [shell] * drop Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
@@ -1,6 +1,7 @@
|
||||
package hfapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -112,6 +113,17 @@ func NewClient() *Client {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) newRequest(ctx context.Context, method, rawURL, token string) (*http.Request, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, method, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// SearchModels searches for models using the Hugging Face API
|
||||
func (c *Client) SearchModels(params SearchParams) ([]Model, error) {
|
||||
for attempt := 1; attempt <= c.maxRetries; attempt++ {
|
||||
|
||||
239
pkg/huggingface-api/snapshot.go
Normal file
239
pkg/huggingface-api/snapshot.go
Normal file
@@ -0,0 +1,239 @@
|
||||
package hfapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultMaxSnapshotFiles = 100_000
|
||||
|
||||
var immutableRevisionPattern = regexp.MustCompile(`^[0-9a-fA-F]{40}$`)
|
||||
|
||||
type SnapshotRequest struct {
|
||||
Repo string
|
||||
Revision string
|
||||
Token string
|
||||
AllowPatterns []string
|
||||
IgnorePatterns []string
|
||||
MaxFiles int
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
Endpoint string
|
||||
Repo string
|
||||
RequestedRevision string
|
||||
ResolvedRevision string
|
||||
Files []SnapshotFile
|
||||
}
|
||||
|
||||
type SnapshotFile struct {
|
||||
Path string
|
||||
Size int64
|
||||
BlobOID string
|
||||
LFSOID string
|
||||
XetHash string
|
||||
URL string
|
||||
}
|
||||
|
||||
func FilterSnapshotFiles(files []SnapshotFile, allow, ignore []string) ([]SnapshotFile, error) {
|
||||
selected := make([]SnapshotFile, 0, len(files))
|
||||
for _, file := range files {
|
||||
allowed, err := matchesAny(file.Path, allow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(allow) > 0 && !allowed {
|
||||
continue
|
||||
}
|
||||
ignored, err := matchesAny(file.Path, ignore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ignored {
|
||||
continue
|
||||
}
|
||||
selected = append(selected, file)
|
||||
}
|
||||
sort.Slice(selected, func(i, j int) bool { return selected[i].Path < selected[j].Path })
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func matchesAny(filePath string, patterns []string) (bool, error) {
|
||||
for _, pattern := range patterns {
|
||||
matched, err := path.Match(pattern, filePath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("invalid Hub file pattern %q: %w", pattern, err)
|
||||
}
|
||||
if !matched && !strings.Contains(pattern, "/") {
|
||||
matched, err = path.Match(pattern, path.Base(filePath))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("invalid Hub file pattern %q: %w", pattern, err)
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type revisionResponse struct {
|
||||
SHA string `json:"sha"`
|
||||
}
|
||||
|
||||
type snapshotTreeItem struct {
|
||||
Type string `json:"type"`
|
||||
OID string `json:"oid"`
|
||||
Size int64 `json:"size"`
|
||||
Path string `json:"path"`
|
||||
LFS *LFSInfo `json:"lfs,omitempty"`
|
||||
XetHash string `json:"xetHash,omitempty"`
|
||||
}
|
||||
|
||||
func (c *Client) ResolveSnapshot(ctx context.Context, req SnapshotRequest) (Snapshot, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if req.MaxFiles <= 0 {
|
||||
req.MaxFiles = defaultMaxSnapshotFiles
|
||||
}
|
||||
endpoint := strings.TrimSuffix(c.baseURL, "/api/models")
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" {
|
||||
return Snapshot{}, fmt.Errorf("invalid Hugging Face endpoint")
|
||||
}
|
||||
endpoint = strings.TrimRight(parsed.String(), "/")
|
||||
owner, repo, err := splitRepo(req.Repo)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
revision := req.Revision
|
||||
if revision == "" {
|
||||
revision = "main"
|
||||
}
|
||||
metadataURL := fmt.Sprintf("%s/api/models/%s/%s/revision/%s", endpoint, url.PathEscape(owner), url.PathEscape(repo), url.PathEscape(revision))
|
||||
metadataReq, err := c.newRequest(ctx, http.MethodGet, metadataURL, req.Token)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
var metadata revisionResponse
|
||||
if err := c.decodeJSON(metadataReq, &metadata); err != nil {
|
||||
return Snapshot{}, fmt.Errorf("resolve Hugging Face revision: %w", err)
|
||||
}
|
||||
if !immutableRevisionPattern.MatchString(metadata.SHA) {
|
||||
return Snapshot{}, fmt.Errorf("Hub returned invalid immutable revision")
|
||||
}
|
||||
resolvedRevision := strings.ToLower(metadata.SHA)
|
||||
|
||||
treeURL := fmt.Sprintf("%s/api/models/%s/%s/tree/%s?recursive=true&expand=true", endpoint, url.PathEscape(owner), url.PathEscape(repo), resolvedRevision)
|
||||
items := make([]snapshotTreeItem, 0)
|
||||
for treeURL != "" {
|
||||
treeReq, err := c.newRequest(ctx, http.MethodGet, treeURL, req.Token)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
page, next, err := c.decodeTreePage(treeReq)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if len(items)+len(page) > req.MaxFiles {
|
||||
return Snapshot{}, fmt.Errorf("snapshot exceeds maximum file count %d", req.MaxFiles)
|
||||
}
|
||||
items = append(items, page...)
|
||||
treeURL = next
|
||||
}
|
||||
|
||||
files := make([]SnapshotFile, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item.Type != "file" {
|
||||
continue
|
||||
}
|
||||
file := SnapshotFile{Path: item.Path, Size: item.Size, BlobOID: item.OID, XetHash: item.XetHash}
|
||||
if item.LFS != nil {
|
||||
file.LFSOID = item.LFS.Oid
|
||||
}
|
||||
file.URL = fmt.Sprintf("%s/%s/%s/resolve/%s/%s", endpoint, url.PathEscape(owner), url.PathEscape(repo), resolvedRevision, escapeFilePath(item.Path))
|
||||
files = append(files, file)
|
||||
}
|
||||
files, err = FilterSnapshotFiles(files, req.AllowPatterns, req.IgnorePatterns)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
return Snapshot{Endpoint: endpoint, Repo: req.Repo, RequestedRevision: revision, ResolvedRevision: resolvedRevision, Files: files}, nil
|
||||
}
|
||||
|
||||
func splitRepo(repo string) (string, string, error) {
|
||||
parts := strings.Split(repo, "/")
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", fmt.Errorf("Hugging Face repo must be owner/repo")
|
||||
}
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
func escapeFilePath(filePath string) string {
|
||||
parts := strings.Split(filePath, "/")
|
||||
for i := range parts {
|
||||
parts[i] = url.PathEscape(parts[i])
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func (c *Client) decodeJSON(req *http.Request, dst any) error {
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("Hub request returned status %d", resp.StatusCode)
|
||||
}
|
||||
return json.NewDecoder(http.MaxBytesReader(nil, resp.Body, 8<<20)).Decode(dst)
|
||||
}
|
||||
|
||||
func (c *Client) decodeTreePage(req *http.Request) ([]snapshotTreeItem, string, error) {
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", fmt.Errorf("Hub tree request returned status %d", resp.StatusCode)
|
||||
}
|
||||
var page []snapshotTreeItem
|
||||
if err := json.NewDecoder(http.MaxBytesReader(nil, resp.Body, 32<<20)).Decode(&page); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
next, err := nextPage(resp.Header.Get("Link"), req.URL)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return page, next, nil
|
||||
}
|
||||
|
||||
func nextPage(link string, current *url.URL) (string, error) {
|
||||
for _, entry := range strings.Split(link, ",") {
|
||||
parts := strings.Split(entry, ";")
|
||||
if len(parts) >= 2 && strings.TrimSpace(parts[1]) == `rel="next"` {
|
||||
raw := strings.Trim(strings.TrimSpace(parts[0]), "<>")
|
||||
next, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid Hub pagination link: %w", err)
|
||||
}
|
||||
if !next.IsAbs() {
|
||||
next = current.ResolveReference(next)
|
||||
}
|
||||
if next.Scheme != current.Scheme || next.Host != current.Host {
|
||||
return "", fmt.Errorf("cross-origin pagination link rejected")
|
||||
}
|
||||
return next.String(), nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
149
pkg/huggingface-api/snapshot_test.go
Normal file
149
pkg/huggingface-api/snapshot_test.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package hfapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
|
||||
)
|
||||
|
||||
var _ = Describe("immutable snapshot resolution", func() {
|
||||
It("pins the metadata sha and follows tree pagination", func() {
|
||||
const commit = "0123456789abcdef0123456789abcdef01234567"
|
||||
var server *httptest.Server
|
||||
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
Expect(r.Header.Get("Authorization")).To(Equal("Bearer test-token"))
|
||||
switch {
|
||||
case strings.Contains(r.URL.Path, "/revision/main"):
|
||||
_, _ = w.Write([]byte(`{"sha":"` + commit + `"}`))
|
||||
case strings.Contains(r.URL.Path, "/tree/"+commit) && r.URL.Query().Get("cursor") == "page-2":
|
||||
_, _ = w.Write([]byte(`[{"type":"file","path":"tokenizer/tokenizer.json","size":7,"oid":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}]`))
|
||||
case strings.Contains(r.URL.Path, "/tree/"+commit):
|
||||
Expect(r.URL.Query().Get("recursive")).To(Equal("true"))
|
||||
w.Header().Set("Link", fmt.Sprintf(`<%s%s?recursive=true&expand=true&cursor=page-2>; rel="next"`, server.URL, r.URL.Path))
|
||||
_, _ = w.Write([]byte(`[{"type":"file","path":"model.safetensors","size":11,"oid":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","lfs":{"oid":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","size":11,"pointerSize":130}}]`))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
DeferCleanup(server.Close)
|
||||
|
||||
client := hfapi.NewClient()
|
||||
client.SetBaseURL(server.URL + "/api/models")
|
||||
snapshot, err := client.ResolveSnapshot(context.Background(), hfapi.SnapshotRequest{
|
||||
Repo: "owner/repo",
|
||||
Revision: "main",
|
||||
Token: "test-token",
|
||||
MaxFiles: 10,
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(snapshot.Endpoint).To(Equal(server.URL))
|
||||
Expect(snapshot.ResolvedRevision).To(Equal(commit))
|
||||
Expect(snapshot.Files).To(HaveLen(2))
|
||||
Expect(snapshot.Files[0].Path).To(Equal("model.safetensors"))
|
||||
Expect(snapshot.Files[0].LFSOID).To(HaveLen(64))
|
||||
Expect(snapshot.Files[1].Path).To(Equal("tokenizer/tokenizer.json"))
|
||||
Expect(snapshot.Files[0].URL).To(ContainSubstring("/resolve/" + commit + "/model.safetensors"))
|
||||
})
|
||||
|
||||
It("applies deterministic allow and ignore filters", func() {
|
||||
files := []hfapi.SnapshotFile{
|
||||
{Path: "config.json"},
|
||||
{Path: "nested/tokenizer.json"},
|
||||
{Path: "weights.bin"},
|
||||
{Path: "weights.safetensors"},
|
||||
}
|
||||
filtered, err := hfapi.FilterSnapshotFiles(files, []string{"*.json", "*.safetensors"}, []string{"nested/*"})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(filtered).To(ConsistOf(
|
||||
hfapi.SnapshotFile{Path: "config.json"},
|
||||
hfapi.SnapshotFile{Path: "weights.safetensors"},
|
||||
))
|
||||
})
|
||||
|
||||
It("rejects a manifest larger than MaxFiles", func() {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.Contains(r.URL.Path, "/revision/") {
|
||||
_, _ = w.Write([]byte(`{"sha":"0123456789abcdef0123456789abcdef01234567"}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"type":"file","path":"a","size":1},{"type":"file","path":"b","size":1}]`))
|
||||
}))
|
||||
DeferCleanup(server.Close)
|
||||
client := hfapi.NewClient()
|
||||
client.SetBaseURL(server.URL + "/api/models")
|
||||
_, err := client.ResolveSnapshot(context.Background(), hfapi.SnapshotRequest{Repo: "owner/repo", Revision: "main", MaxFiles: 1})
|
||||
Expect(err).To(MatchError(ContainSubstring("maximum file count")))
|
||||
})
|
||||
|
||||
It("honors a cancelled context before metadata access", func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := hfapi.NewClient().ResolveSnapshot(ctx, hfapi.SnapshotRequest{Repo: "owner/repo", Revision: "main"})
|
||||
Expect(err).To(MatchError(context.Canceled))
|
||||
})
|
||||
|
||||
It("rejects a cross-origin pagination link before sending credentials", func() {
|
||||
const commit = "0123456789abcdef0123456789abcdef01234567"
|
||||
requestedCrossOrigin := false
|
||||
evil := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
requestedCrossOrigin = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
DeferCleanup(evil.Close)
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.Contains(r.URL.Path, "/revision/") {
|
||||
_, _ = w.Write([]byte(`{"sha":"` + commit + `"}`))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Link", `<`+evil.URL+`/steal>; rel="next"`)
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
}))
|
||||
DeferCleanup(origin.Close)
|
||||
client := hfapi.NewClient()
|
||||
client.SetBaseURL(origin.URL + "/api/models")
|
||||
_, err := client.ResolveSnapshot(context.Background(), hfapi.SnapshotRequest{
|
||||
Repo: "owner/repo", Revision: "main", Token: "test-token",
|
||||
})
|
||||
Expect(err).To(MatchError(ContainSubstring("cross-origin pagination")))
|
||||
Expect(requestedCrossOrigin).To(BeFalse())
|
||||
})
|
||||
|
||||
It("downloads a pinned public Xet fixture through the ordinary resolve URL", Label("network"), func() {
|
||||
if os.Getenv("LOCALAI_HF_XET_SMOKE") != "1" {
|
||||
Skip("set LOCALAI_HF_XET_SMOKE=1 to run the public Hub compatibility gate")
|
||||
}
|
||||
client := hfapi.NewClient()
|
||||
snapshot, err := client.ResolveSnapshot(context.Background(), hfapi.SnapshotRequest{
|
||||
Repo: "hf-internal-testing/tiny-random-t5-v1.1",
|
||||
Revision: "main",
|
||||
AllowPatterns: []string{"model.safetensors"},
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(snapshot.ResolvedRevision).To(MatchRegexp(`^[0-9a-f]{40}$`))
|
||||
Expect(snapshot.Files).To(HaveLen(1))
|
||||
file := snapshot.Files[0]
|
||||
Expect(file.Path).To(Equal("model.safetensors"))
|
||||
Expect(file.XetHash).NotTo(BeEmpty())
|
||||
target := filepath.Join(GinkgoT().TempDir(), file.Path)
|
||||
var final downloader.TransferProgress
|
||||
err = downloader.URI(file.URL).DownloadFileWithContext(
|
||||
context.Background(), target, file.LFSOID, 0, 1, nil,
|
||||
downloader.WithTransferProgress(func(event downloader.TransferProgress) { final = event }),
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
info, err := os.Stat(target)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(info.Size()).To(Equal(file.Size))
|
||||
Expect(final.Written).To(Equal(file.Size))
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user