mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
feat: materialize Hugging Face model artifacts (#10825)
* feat(config): add model artifact source contract Assisted-by: Codex:GPT-5 [Codex] * feat(downloader): add authenticated raw-byte progress Assisted-by: Codex:GPT-5 [Codex] * feat(huggingface): resolve immutable snapshot manifests Assisted-by: Codex:GPT-5 [Codex] * feat(models): add artifact storage primitives Assisted-by: Codex:GPT-5 [Codex] * feat(models): materialize pinned Hugging Face snapshots Assisted-by: Codex:GPT-5 [Codex] * feat(models): bind managed snapshots at runtime Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): materialize model artifacts during install Assisted-by: Codex:GPT-5 [Codex] * feat(gallery): declare managed Hugging Face artifacts Assisted-by: Codex:GPT-5 [Codex] * feat(models): preload managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * fix(gallery): retain shared artifact caches on delete Assisted-by: Codex:GPT-5 [Codex] * feat(models): report artifact acquisition progress Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load managed models from ModelFile Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): load staged speech model snapshots Assisted-by: Codex:GPT-5 [Codex] * refactor(backends): use staged snapshots in engine backends Assisted-by: Codex:GPT-5 [Codex] * test(distributed): cover staged artifact snapshots Assisted-by: Codex:GPT-5 [Codex] * docs: explain managed model artifacts Assisted-by: Codex:GPT-5 [Codex] * docs: add product design context Assisted-by: Codex:GPT-5 [Codex] * feat(ui): show model artifact download progress Assisted-by: Codex:GPT-5 [Codex] * Eagerly materialize Hugging Face artifacts Materialize HF-backed model references as managed GGUF artifacts during load, with lazy download retained only as fallback. Assisted-by: Codex:GPT-5 [shell] * Refactor HF downloads through a shared executor Assisted-by: Codex:GPT-5 [shell] * drop Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
@@ -19,6 +19,7 @@ import grpc
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
|
||||
from grpc_auth import get_auth_interceptors
|
||||
from model_utils import resolve_model_reference
|
||||
|
||||
import torch
|
||||
import torch.cuda
|
||||
@@ -61,11 +62,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
|
||||
|
||||
def LoadModel(self, request, context):
|
||||
model_name = request.Model
|
||||
|
||||
# Check to see if the Model exists in the filesystem already.
|
||||
if os.path.exists(request.ModelFile):
|
||||
model_name = request.ModelFile
|
||||
model_name, local_only = resolve_model_reference(request)
|
||||
|
||||
compute = torch.float16
|
||||
if request.F16Memory == True:
|
||||
@@ -150,7 +147,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
device_map=device_map,
|
||||
load_in_4bit=xpu_4bit,
|
||||
load_in_8bit=xpu_8bit,
|
||||
torch_dtype=compute)
|
||||
torch_dtype=compute,
|
||||
local_files_only=local_only)
|
||||
elif request.Type == "OVModelForCausalLM":
|
||||
from optimum.intel.openvino import OVModelForCausalLM
|
||||
from openvino.runtime import Core
|
||||
@@ -171,7 +169,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
compile=True,
|
||||
trust_remote_code=request.TrustRemoteCode,
|
||||
ov_config=ovconfig,
|
||||
device=device_map)
|
||||
device=device_map,
|
||||
local_files_only=local_only)
|
||||
self.OV = True
|
||||
elif request.Type == "OVModelForFeatureExtraction":
|
||||
from optimum.intel.openvino import OVModelForFeatureExtraction
|
||||
@@ -194,11 +193,16 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
trust_remote_code=request.TrustRemoteCode,
|
||||
ov_config=ovconfig,
|
||||
export=True,
|
||||
device=device_map)
|
||||
device=device_map,
|
||||
local_files_only=local_only)
|
||||
self.OV = True
|
||||
elif request.Type == "SentenceTransformer":
|
||||
autoTokenizer = False
|
||||
self.model = SentenceTransformer(model_name, trust_remote_code=request.TrustRemoteCode)
|
||||
self.model = SentenceTransformer(
|
||||
model_name,
|
||||
trust_remote_code=request.TrustRemoteCode,
|
||||
local_files_only=local_only,
|
||||
)
|
||||
self.SentenceTransformer = True
|
||||
elif request.Type == "TokenClassification":
|
||||
# NER / PII tagging via HuggingFace's token-classification
|
||||
@@ -213,6 +217,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
aggregation_strategy="simple",
|
||||
device=0 if self.CUDA else -1,
|
||||
trust_remote_code=request.TrustRemoteCode,
|
||||
model_kwargs={"local_files_only": local_only},
|
||||
)
|
||||
self.TokenClassification = True
|
||||
else:
|
||||
@@ -231,6 +236,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
quantization_config=quantization,
|
||||
device_map=device_map,
|
||||
torch_dtype=compute,
|
||||
local_files_only=local_only,
|
||||
)
|
||||
|
||||
# Try to load a processor (needed for TTS/audio models)
|
||||
@@ -238,6 +244,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
self.processor = AutoProcessor.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code=request.TrustRemoteCode,
|
||||
local_files_only=local_only,
|
||||
)
|
||||
self.GenericTTS = True
|
||||
print(f"Loaded processor for {model_name}", file=sys.stderr)
|
||||
@@ -252,7 +259,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
self.max_tokens = self.options.get("max_new_tokens", 512)
|
||||
|
||||
if autoTokenizer:
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
local_files_only=local_only,
|
||||
)
|
||||
self.XPU = False
|
||||
|
||||
if XPU and self.OV == False:
|
||||
|
||||
Reference in New Issue
Block a user